X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=ee1dcdef922a884cbe63d800584ef574fe5ace2a;hb=066d01be84db56933591117e616ac35203dffd18;hp=73619d590395bb00f189fbdc6e5b7f0bd377fbaa;hpb=b11e4226340fc584ec9ee8199005d71d60e7d650;p=dak.git diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 73619d59..ee1dcdef 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -1,7 +1,13 @@ #!/usr/bin/env python -""" Check for obsolete binary packages """ -# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup +""" +Check for obsolete binary packages + +@contact: Debian FTP Master +@copyright: 2000-2006 James Troup +@copyright: 2009 Torsten Werner +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -192,10 +198,14 @@ def do_newer_version(lowersuite_name, highersuite_name, code, session): # Check for packages in $highersuite obsoleted by versions in $lowersuite q = session.execute(""" +WITH highersuite_maxversion AS (SELECT s.source AS source, max(s.version) AS version + FROM src_associations sa, source s + WHERE sa.suite = :highersuite_id AND sa.source = s.id group by s.source) SELECT s.source, s.version AS lower, s2.version AS higher - FROM src_associations sa, source s, source s2, src_associations sa2 + FROM src_associations sa, source s, source s2, src_associations sa2, highersuite_maxversion hm WHERE sa.suite = :highersuite_id AND sa2.suite = :lowersuite_id AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source + AND hm.source = s.source AND hm.version < s2.version AND s.version < s2.version""", {'lowersuite_id': lowersuite.suite_id, 'highersuite_id': highersuite.suite_id}) ql = q.fetchall() @@ -216,36 +226,183 @@ SELECT s.source, s.version AS lower, s2.version AS higher ################################################################################ -def do_nbs(real_nbs): - output = "Not Built from Source\n" - output += "---------------------\n\n" - - cmd_output = "" - nbs_keys = real_nbs.keys() - nbs_keys.sort() - for source in nbs_keys: - output += " * %s_%s builds: %s\n" % (source, - source_versions.get(source, "??"), - source_binaries.get(source, "(source does not exist)")) - output += " but no longer builds:\n" - versions = real_nbs[source].keys() - versions.sort(apt_pkg.VersionCompare) - all_packages = [] - for version in versions: - packages = real_nbs[source][version].keys() - packages.sort() - all_packages.extend(packages) - output += " o %s: %s\n" % (version, ", ".join(packages)) - if all_packages: - all_packages.sort() - cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s -R\n\n" % (source, suite.suite_name, " ".join(all_packages)) - - output += "\n" - - if len(cmd_output): - print output - print "Suggested commands:\n" - print cmd_output +def queryWithoutSource(suite_id, session): + """searches for arch: all packages from suite that do no longer + reference a source package in the same suite + + subquery unique_binaries: selects all packages with only 1 version + in suite since 'dak rm' does not allow to specify version numbers""" + + query = """ + with unique_binaries as + (select package, max(version) as version, max(source) as source + from bin_associations_binaries + where architecture = 2 and suite = :suite_id + group by package having count(*) = 1) + select ub.package, ub.version + from unique_binaries ub + left join src_associations_src sas + on ub.source = sas.src and sas.suite = :suite_id + where sas.id is null + order by ub.package""" + return session.execute(query, { 'suite_id': suite_id }) + +def reportWithoutSource(suite_name, suite_id, session): + rows = queryWithoutSource(suite_id, session) + title = 'packages without source in suite %s' % suite_name + if rows.rowcount > 0: + print '%s\n%s\n' % (title, '-' * len(title)) + message = '"[auto-cruft] no longer built from source"' + for row in rows: + (package, version) = row + print "* package %s in version %s is no longer built from source" % \ + (package, version) + print " - suggested command:" + print " dak rm -m %s -s %s -a all -p -R -b %s\n" % \ + (message, suite_name, package) + +def queryNewerAll(suite_name, session): + """searches for arch != all packages that have an arch == all + package with a higher version in the same suite""" + + query = """ +select bab1.package, bab1.version as oldver, + array_to_string(array_agg(a.arch_string), ',') as oldarch, + bab2.version as newver + from bin_associations_binaries bab1 + join bin_associations_binaries bab2 + on bab1.package = bab2.package and bab1.version < bab2.version and + bab1.suite = bab2.suite and bab1.architecture > 2 and + bab2.architecture = 2 + join architecture a on bab1.architecture = a.id + join suite s on bab1.suite = s.id + where s.suite_name = :suite_name + group by bab1.package, oldver, bab1.suite, newver""" + return session.execute(query, { 'suite_name': suite_name }) + +def reportNewerAll(suite_name, session): + rows = queryNewerAll(suite_name, session) + title = 'obsolete arch any packages in suite %s' % suite_name + if rows.rowcount > 0: + print '%s\n%s\n' % (title, '-' * len(title)) + message = '"[auto-cruft] obsolete arch any package"' + for row in rows: + (package, oldver, oldarch, newver) = row + print "* package %s is arch any in version %s but arch all in version %s" % \ + (package, oldver, newver) + print " - suggested command:" + print " dak rm -m %s -s %s -a %s -p -b %s\n" % \ + (message, suite_name, oldarch, package) + +def queryNBS(suite_id, session): + """This one is really complex. It searches arch != all packages that + are no longer built from current source packages in suite. + + temp table unique_binaries: will be populated with packages that + have only one version in suite because 'dak rm' does not allow + specifying version numbers + + temp table newest_binaries: will be populated with packages that are + built from current sources + + subquery uptodate_arch: returns all architectures built from current + sources + + subquery unique_binaries_uptodate_arch: returns all packages in + architectures from uptodate_arch + + subquery unique_binaries_uptodate_arch_agg: same as + unique_binaries_uptodate_arch but with column architecture + aggregated to array + + subquery uptodate_packages: similar to uptodate_arch but returns all + packages built from current sources + + subquery outdated_packages: returns all packages with architectures + no longer built from current source + """ + + query = """ +create temp table unique_binaries ( + package text not null, + architecture integer not null, + source integer not null); + +insert into unique_binaries + select bab.package, bab.architecture, max(bab.source) + from bin_associations_binaries bab + where bab.suite = :suite_id and bab.architecture > 2 + group by package, architecture having count(*) = 1; + +create temp table newest_binaries ( + package text not null, + architecture integer not null, + source text not null, + version debversion not null); + +insert into newest_binaries + select ub.package, ub.architecture, nsa.source, nsa.version + from unique_binaries ub + join newest_src_association nsa + on ub.source = nsa.src and nsa.suite = :suite_id; + +with uptodate_arch as + (select architecture, source, version + from newest_binaries + group by architecture, source, version), + unique_binaries_uptodate_arch as + (select ub.package, ub.architecture, ua.source, ua.version + from unique_binaries ub + join source s + on ub.source = s.id + join uptodate_arch ua + on ub.architecture = ua.architecture and s.source = ua.source), + unique_binaries_uptodate_arch_agg as + (select ubua.package, + array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list, + ubua.source, ubua.version + from unique_binaries_uptodate_arch ubua + join architecture a + on ubua.architecture = a.id + group by ubua.source, ubua.version, ubua.package), + uptodate_packages as + (select package, source, version + from newest_binaries + group by package, source, version), + outdated_packages as + (select array(select unnest(array_agg(package)) order by 1) as pkg_list, + arch_list, source, version + from unique_binaries_uptodate_arch_agg + where package not in + (select package from uptodate_packages) + group by arch_list, source, version) + select * from outdated_packages order by source""" + return session.execute(query, { 'suite_id': suite_id }) + +def reportNBS(suite_name, suite_id): + session = DBConn().session() + nbsRows = queryNBS(suite_id, session) + title = 'NBS packages in suite %s' % suite_name + if nbsRows.rowcount > 0: + print '%s\n%s\n' % (title, '-' * len(title)) + for row in nbsRows: + (pkg_list, arch_list, source, version) = row + pkg_string = ' '.join(pkg_list) + arch_string = ','.join(arch_list) + print "* source package %s version %s no longer builds" % \ + (source, version) + print " binary package(s): %s" % pkg_string + print " on %s" % arch_string + print " - suggested command:" + message = '"[auto-cruft] NBS (no longer built by %s)"' % source + print " dak rm -m %s -s %s -a %s -p -R -b %s\n" % \ + (message, suite_name, arch_string, pkg_string) + session.close() + +def reportAllNBS(suite_name, suite_id, session): + reportWithoutSource(suite_name, suite_id, session) + reportNewerAll(suite_name, session) + reportNBS(suite_name, suite_id) ################################################################################ @@ -272,47 +429,84 @@ def do_dubious_nbs(dubious_nbs): ################################################################################ -def do_obsolete_source(duplicate_bins, bin2source): - obsolete = {} - for key in duplicate_bins.keys(): - (source_a, source_b) = key.split('_') - for source in [ source_a, source_b ]: - if not obsolete.has_key(source): - if not source_binaries.has_key(source): - # Source has already been removed - continue - else: - obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ] - for binary in duplicate_bins[key]: - if bin2source.has_key(binary) and bin2source[binary]["source"] == source: - continue - if binary in obsolete[source]: - obsolete[source].remove(binary) - - to_remove = [] - output = "Obsolete source package\n" - output += "-----------------------\n\n" - obsolete_keys = obsolete.keys() - obsolete_keys.sort() - for source in obsolete_keys: - if not obsolete[source]: - to_remove.append(source) - output += " * %s (%s)\n" % (source, source_versions[source]) - for binary in [ i.strip() for i in source_binaries[source].split(',') ]: - if bin2source.has_key(binary): - output += " o %s (%s) is built by %s.\n" \ - % (binary, bin2source[binary]["version"], - bin2source[binary]["source"]) - else: - output += " o %s is not built.\n" % binary - output += "\n" - - if to_remove: - print output - - print "Suggested command:" - print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove)) - print +def obsolete_source(suite_name, session): + """returns obsolete source packages for suite_name without binaries + in the same suite sorted by install_date; install_date should help + detecting source only (or binary throw away) uploads; duplicates in + the suite are skipped + + subquery 'source_suite_unique' returns source package names from + suite without duplicates; the rationale behind is that neither + cruft-report nor rm cannot handle duplicates (yet)""" + + query = """ +WITH source_suite_unique AS + (SELECT source, suite + FROM source_suite GROUP BY source, suite HAVING count(*) = 1) +SELECT ss.src, ss.source, ss.version, + to_char(ss.install_date, 'YYYY-MM-DD') AS install_date + FROM source_suite ss + JOIN source_suite_unique ssu + ON ss.source = ssu.source AND ss.suite = ssu.suite + JOIN suite s ON s.id = ss.suite + LEFT JOIN bin_associations_binaries bab + ON ss.src = bab.source AND ss.suite = bab.suite + WHERE s.suite_name = :suite_name AND bab.id IS NULL + ORDER BY install_date""" + args = { 'suite_name': suite_name } + return session.execute(query, args) + +def source_bin(source, session): + """returns binaries built by source for all or no suite grouped and + ordered by package name""" + + query = """ +SELECT b.package + FROM binaries b + JOIN src_associations_src sas ON b.source = sas.src + WHERE sas.source = :source + GROUP BY b.package + ORDER BY b.package""" + args = { 'source': source } + return session.execute(query, args) + +def newest_source_bab(suite_name, package, session): + """returns newest source that builds binary package in suite grouped + and sorted by source and package name""" + + query = """ +SELECT sas.source, MAX(sas.version) AS srcver + FROM src_associations_src sas + JOIN bin_associations_binaries bab ON sas.src = bab.source + JOIN suite s on s.id = bab.suite + WHERE s.suite_name = :suite_name AND bab.package = :package + GROUP BY sas.source, bab.package + ORDER BY sas.source, bab.package""" + args = { 'suite_name': suite_name, 'package': package } + return session.execute(query, args) + +def report_obsolete_source(suite_name, session): + rows = obsolete_source(suite_name, session) + if rows.rowcount == 0: + return + print \ +"""Obsolete source packages in suite %s +----------------------------------%s\n""" % \ + (suite_name, '-' * len(suite_name)) + for os_row in rows.fetchall(): + (src, old_source, version, install_date) = os_row + print " * obsolete source %s version %s installed at %s" % \ + (old_source, version, install_date) + for sb_row in source_bin(old_source, session): + (package, ) = sb_row + print " - has built binary %s" % package + for nsb_row in newest_source_bab(suite_name, package, session): + (new_source, srcver) = nsb_row + print " currently built by source %s version %s" % \ + (new_source, srcver) + print " - suggested command:" + rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\"" + print " dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source) def get_suite_binaries(suite, session): # Initalize a large hash table of all binary packages @@ -358,9 +552,7 @@ def main (): # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "nvit" ] - # 'obsolete source' is broken since the introduction of dak dominate - #checks = [ "nbs", "nviu", "nvit", "obsolete source" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] else: @@ -385,6 +577,12 @@ def main (): suite_id = suite.suite_id suite_name = suite.suite_name.lower() + if "obsolete source" in checks: + report_obsolete_source(suite_name, session) + + if "nbs" in checks: + reportAllNBS(suite_name, suite_id, session) + bin_not_built = {} if "bnb" in checks: @@ -503,12 +701,8 @@ def main (): packages.close() os.unlink(temp_filename) - if "obsolete source" in checks: - do_obsolete_source(duplicate_bins, bin2source) - # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} - real_nbs = {} for source in nbs.keys(): for package in nbs[source].keys(): versions = nbs[source][package].keys() @@ -517,8 +711,6 @@ def main (): source_version = source_versions.get(source,"0") if apt_pkg.VersionCompare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) - else: - add_nbs(real_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) @@ -526,9 +718,6 @@ def main (): if "nvit" in checks: do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) - if "nbs" in checks: - do_nbs(real_nbs) - ### if Options["Mode"] == "full":