From: Torsten Werner Date: Fri, 27 Nov 2009 22:55:04 +0000 (+0100) Subject: cruft-report: reimplement 'obsolete source' X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=2e37e696ba3b6eb5401727a2dad2288687bb2f5b;p=dak.git cruft-report: reimplement 'obsolete source' Signed-off-by: Torsten Werner --- diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 73619d59..384ad0e7 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -272,47 +272,64 @@ def do_dubious_nbs(dubious_nbs): ################################################################################ -def do_obsolete_source(duplicate_bins, bin2source): - obsolete = {} - for key in duplicate_bins.keys(): - (source_a, source_b) = key.split('_') - for source in [ source_a, source_b ]: - if not obsolete.has_key(source): - if not source_binaries.has_key(source): - # Source has already been removed - continue - else: - obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ] - for binary in duplicate_bins[key]: - if bin2source.has_key(binary) and bin2source[binary]["source"] == source: - continue - if binary in obsolete[source]: - obsolete[source].remove(binary) - - to_remove = [] - output = "Obsolete source package\n" - output += "-----------------------\n\n" - obsolete_keys = obsolete.keys() - obsolete_keys.sort() - for source in obsolete_keys: - if not obsolete[source]: - to_remove.append(source) - output += " * %s (%s)\n" % (source, source_versions[source]) - for binary in [ i.strip() for i in source_binaries[source].split(',') ]: - if bin2source.has_key(binary): - output += " o %s (%s) is built by %s.\n" \ - % (binary, bin2source[binary]["version"], - bin2source[binary]["source"]) - else: - output += " o %s is not built.\n" % binary - output += "\n" - - if to_remove: - print output - - print "Suggested command:" - print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove)) - print +def obsolete_source(suite_name, session): + """returns obsolete source packages for suite_name sorted by + install_date""" + + query = """ +SELECT os.src, os.source, os.version, os.install_date + FROM obsolete_source os + JOIN suite s on s.id = os.suite + WHERE s.suite_name = :suite_name + ORDER BY install_date""" + args = { 'suite_name': suite_name } + return session.execute(query, args) + +def source_bin(source, session): + """returns binaries built by source for all or no suite""" + + query = """ +SELECT package + FROM source_bin + WHERE source = :source + ORDER BY package""" + args = { 'source': source } + return session.execute(query, args) + +def newest_source_bab(suite_name, package, session): + """returns newest source that builds binary package in suite""" + + query = """ +SELECT source, srcver + FROM newest_source_bab nsb + JOIN suite s on s.id = nsb.suite + WHERE s.suite_name = :suite_name AND nsb.package = :package + ORDER BY source""" + args = { 'suite_name': suite_name, 'package': package } + return session.execute(query, args) + +def report_obsolete_source(suite_name, session): + rows = obsolete_source(suite_name, session) + if rows.rowcount == 0: + return + print \ +"""Obsolete source packages in suite %s +----------------------------------%s\n""" % \ + (suite_name, '-' * len(suite_name)) + for os_row in rows.fetchall(): + (src, old_source, version, install_date) = os_row + print " * obsolete source %s version %s installed at %s" % \ + (old_source, version, install_date) + for sb_row in source_bin(old_source, session): + (package, ) = sb_row + print " - has built binary %s" % package + for nsb_row in newest_source_bab(suite_name, package, session): + (new_source, srcver) = nsb_row + print " currently built by source %s version %s" % \ + (new_source, srcver) + print " - suggested command:" + rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\"" + print " dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source) def get_suite_binaries(suite, session): # Initalize a large hash table of all binary packages @@ -358,9 +375,7 @@ def main (): # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "nvit" ] - # 'obsolete source' is broken since the introduction of dak dominate - #checks = [ "nbs", "nviu", "nvit", "obsolete source" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] else: @@ -385,6 +400,9 @@ def main (): suite_id = suite.suite_id suite_name = suite.suite_name.lower() + if "obsolete source" in checks: + report_obsolete_source(suite_name, session) + bin_not_built = {} if "bnb" in checks: @@ -503,9 +521,6 @@ def main (): packages.close() os.unlink(temp_filename) - if "obsolete source" in checks: - do_obsolete_source(duplicate_bins, bin2source) - # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} real_nbs = {} diff --git a/dak/dakdb/update27.py b/dak/dakdb/update27.py new file mode 100755 index 00000000..ee8f5844 --- /dev/null +++ b/dak/dakdb/update27.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +""" +Add views for new obsolete source detection. + +@contact: Debian FTP Master +@copyright: 2009 Torsten Werner +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import psycopg2 + +def do_update(self): + print "Add/modify views for obsolete source detection." + + try: + c = self.db.cursor() + + print "Replace old views." + # joins src_associations and source + c.execute(""" +CREATE OR REPLACE VIEW source_suite AS + SELECT src_associations.id, source.id AS src, source.source, source.version, + src_associations.suite, suite.suite_name, source.install_date + FROM source + JOIN src_associations ON source.id = src_associations.source + JOIN suite ON suite.id = src_associations.suite; + """) + # joins bin_associations and binaries + c.execute(""" +CREATE OR REPLACE VIEW bin_associations_binaries AS + SELECT bin_associations.id, bin_associations.bin, binaries.package, + binaries.version, bin_associations.suite, binaries.architecture, + binaries.source + FROM bin_associations + JOIN binaries ON bin_associations.bin = binaries.id; + """) + + print "Drop old views." + c.execute("DROP VIEW IF EXISTS source_suite_unique CASCADE") + c.execute("DROP VIEW IF EXISTS obsolete_source CASCADE") + c.execute("DROP VIEW IF EXISTS source_bin CASCADE") + c.execute("DROP VIEW IF EXISTS newest_source_bab CASCADE") + + print "Create new views." + # returns source package names from suite without duplicates; + # rationale: cruft-report and rm cannot handle duplicates (yet) + c.execute(""" +CREATE VIEW source_suite_unique + AS SELECT source, suite + FROM source_suite GROUP BY source, suite HAVING count(*) = 1; + """) + # returns obsolete sources without binaries in the same suite; + # outputs install_date to detect source only (or binary throw away) + # uploads; duplicates are skipped + c.execute(""" +CREATE VIEW obsolete_source + AS SELECT ss.src, ss.source, ss.version, ss.suite, + to_char(ss.install_date, 'YYYY-MM-DD') AS install_date + FROM source_suite ss + JOIN source_suite_unique ssu + ON ss.source = ssu.source AND ss.suite = ssu.suite + LEFT JOIN bin_associations_binaries bab + ON ss.src = bab.source AND ss.suite = bab.suite + WHERE bab.id IS NULL; + """) + # returns source package names and its binaries from any suite + c.execute(""" +CREATE VIEW source_bin + AS SELECT b.package, MAX(b.version) AS version, sas.source + FROM binaries b + JOIN src_associations_src sas + ON b.source = sas.src + GROUP BY b.package, sas.source + """) + # returns binaries from suite and their source with max(version) + # grouped by source name, binary name, and suite + c.execute(""" +CREATE VIEW newest_source_bab + AS SELECT sas.source, MAX(sas.version) AS srcver, bab.package, bab.suite + FROM src_associations_src sas + JOIN bin_associations_binaries bab ON sas.src = bab.source + GROUP BY sas.source, bab.package, bab.suite; + """) + + print "Grant permissions to views." + c.execute("GRANT SELECT ON binfiles_suite_component_arch TO PUBLIC;"); + c.execute("GRANT SELECT ON srcfiles_suite_component TO PUBLIC;"); + c.execute("GRANT SELECT ON binaries_suite_arch TO PUBLIC;"); + c.execute("GRANT SELECT ON newest_all_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON obsolete_any_by_all_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON newest_any_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON obsolete_any_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON source_suite TO PUBLIC;"); + c.execute("GRANT SELECT ON newest_source TO PUBLIC;"); + c.execute("GRANT SELECT ON newest_src_association TO PUBLIC;"); + c.execute("GRANT SELECT ON any_associations_source TO PUBLIC;"); + c.execute("GRANT SELECT ON src_associations_src TO PUBLIC;"); + c.execute("GRANT SELECT ON almost_obsolete_src_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON obsolete_src_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON bin_associations_binaries TO PUBLIC;"); + c.execute("GRANT SELECT ON src_associations_bin TO PUBLIC;"); + c.execute("GRANT SELECT ON almost_obsolete_all_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON obsolete_all_associations TO PUBLIC;"); + c.execute("GRANT SELECT ON source_suite_unique TO PUBLIC;"); + c.execute("GRANT SELECT ON obsolete_source TO PUBLIC;"); + c.execute("GRANT SELECT ON source_bin TO PUBLIC;"); + c.execute("GRANT SELECT ON newest_source_bab TO PUBLIC;"); + + print "Committing" + c.execute("UPDATE config SET value = '27' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.InternalError, msg: + self.db.rollback() + raise DBUpdateError, "Database error, rollback issued. Error message : %s" % (str(msg)) + diff --git a/dak/update_db.py b/dak/update_db.py index e7b1984b..5d1cb86c 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError ################################################################################ Cnf = None -required_database_schema = 26 +required_database_schema = 27 ################################################################################