X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fcheck_overrides.py;h=31833198fe88ffeee2e176a068990d6a6ab72fcd;hp=0915b8d40aec6271be6134a99022fc28cb5cbd88;hb=026bdb9580e6d18bf504d2f44d46890df5d89f1a;hpb=06ce98c8111a8b09e5603dbbd34324a216412d69 diff --git a/dak/check_overrides.py b/dak/check_overrides.py index 0915b8d4..31833198 100755 --- a/dak/check_overrides.py +++ b/dak/check_overrides.py @@ -1,8 +1,14 @@ #!/usr/bin/env python -# Cruft checker and hole filler for overrides -# Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup -# Copyright (C) 2005 Jeroen van Wolffelaar +""" Cruft checker and hole filler for overrides + +@contact: Debian FTPMaster +@copyright: 2000, 2001, 2002, 2004, 2006 James Troup +@opyright: 2005 Jeroen van Wolffelaar +@copyright: 2011 Joerg Jaspert +@license: GNU General Public License version 2 or later + +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,15 +54,18 @@ ################################################################################ -import pg, sys, os -import dak.lib.utils, dak.lib.database, dak.lib.logging +import sys, os import apt_pkg +from daklib.config import Config +from daklib.dbconn import * +from daklib import daklog +from daklib import utils + ################################################################################ -Options = None -projectB = None -Logger = None +Options = None #: Commandline arguments parsed into this +Logger = None #: Our logging object sections = {} priorities = {} blacklist = {} @@ -74,274 +83,309 @@ Check for cruft in overrides. ################################################################################ -def gen_blacklist(dir): - for entry in os.listdir(dir): - entry = entry.split('_')[0] - blacklist[entry] = 1 +def process(osuite, affected_suites, originosuite, component, otype, session): + global Logger, Options, sections, priorities -def process(osuite, affected_suites, originosuite, component, type): - global Logger, Options, projectB, sections, priorities + o = get_suite(osuite, session) + if o is None: + utils.fubar("Suite '%s' not recognised." % (osuite)) + osuite_id = o.suite_id - osuite_id = dak.lib.database.get_suite_id(osuite) - if osuite_id == -1: - dak.lib.utils.fubar("Suite '%s' not recognised." % (osuite)) originosuite_id = None if originosuite: - originosuite_id = dak.lib.database.get_suite_id(originosuite) - if originosuite_id == -1: - dak.lib.utils.fubar("Suite '%s' not recognised." % (originosuite)) + oo = get_suite(originosuite, session) + if oo is None: + utils.fubar("Suite '%s' not recognised." % (originosuite)) + originosuite_id = oo.suite_id - component_id = dak.lib.database.get_component_id(component) - if component_id == -1: - dak.lib.utils.fubar("Component '%s' not recognised." % (component)) + c = get_component(component, session) + if c is None: + utils.fubar("Component '%s' not recognised." % (component)) + component_id = c.component_id - type_id = dak.lib.database.get_override_type_id(type) - if type_id == -1: - dak.lib.utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)) - dsc_type_id = dak.lib.database.get_override_type_id("dsc") - deb_type_id = dak.lib.database.get_override_type_id("deb") + ot = get_override_type(otype, session) + if ot is None: + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)) + type_id = ot.overridetype_id + dsc_type_id = get_override_type("dsc", session).overridetype_id - source_priority_id = dak.lib.database.get_priority_id("source") + source_priority_id = get_priority("source", session).priority_id - if type == "deb" or type == "udeb": + if otype == "deb" or otype == "udeb": packages = {} - q = projectB.query(""" -SELECT b.package FROM binaries b, bin_associations ba, files f, - location l, component c - WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location - AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s -""" % (type, ",".join(map(str,affected_suites)), component_id)) - for i in q.getresult(): + # TODO: Fix to use placeholders (check how to with arrays) + q = session.execute(""" +SELECT b.package + FROM binaries b + JOIN bin_associations ba ON b.id = ba.bin + JOIN suite ON ba.suite = suite.id + JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id + WHERE b.type = :otype AND ba.suite IN (%s) AND af.component_id = :component_id +""" % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id}) + for i in q.fetchall(): packages[i[0]] = 0 src_packages = {} - q = projectB.query(""" -SELECT s.source FROM source s, src_associations sa, files f, location l, - component c - WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location - AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s -""" % (",".join(map(str,affected_suites)), component_id)) - for i in q.getresult(): + q = session.execute(""" +SELECT s.source FROM source s + JOIN src_associations sa ON s.id = sa.source + JOIN suite ON sa.suite = suite.id + JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id + WHERE sa.suite IN (%s) AND af.component_id = :component_id +""" % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id}) + for i in q.fetchall(): src_packages[i[0]] = 0 # ----------- # Drop unused overrides - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)) - projectB.query("BEGIN WORK") - if type == "dsc": - for i in q.getresult(): + q = session.execute("""SELECT package, priority, section, maintainer + FROM override WHERE suite = :suite_id + AND component = :component_id AND type = :type_id""", + {'suite_id': osuite_id, 'component_id': component_id, + 'type_id': type_id}) + # We're already within a transaction + if otype == "dsc": + for i in q.fetchall(): package = i[0] if src_packages.has_key(package): src_packages[package] = 1 else: if blacklist.has_key(package): - dak.lib.utils.warn("%s in incoming, not touching" % package) + utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, - type, package, priorities[i[1]], sections[i[2]], i[3]]) + otype, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""DELETE FROM override WHERE package = - '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)) + session.execute("""DELETE FROM override WHERE package = :package + AND suite = :suite_id AND component = :component_id + AND type = :type_id + AND created < now() - interval '14 days'""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': type_id}) # create source overrides based on binary overrides, as source # overrides not always get created - q = projectB.query(""" SELECT package, priority, section, - maintainer FROM override WHERE suite = %s AND component = %s - """ % (osuite_id, component_id)) - for i in q.getresult(): + q = session.execute("""SELECT package, priority, section, maintainer + FROM override WHERE suite = :suite_id AND component = :component_id""", + {'suite_id': osuite_id, 'component_id': component_id}) + for i in q.fetchall(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: continue src_packages[package] = 1 - + Logger.log(["add missing override", osuite, component, - type, package, "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, - osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': source_priority_id, + 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) # Check whether originosuite has an override for us we can # copy if originosuite: - q = projectB.query("""SELECT origin.package, origin.priority, - origin.section, origin.maintainer, target.priority, - target.section, target.maintainer FROM override origin LEFT - JOIN override target ON (origin.package = target.package AND - target.suite=%s AND origin.component = target.component AND origin.type = - target.type) WHERE origin.suite = %s AND origin.component = %s - AND origin.type = %s""" % - (osuite_id, originosuite_id, component_id, type_id)) - for i in q.getresult(): + q = session.execute("""SELECT origin.package, origin.priority, origin.section, + origin.maintainer, target.priority, target.section, + target.maintainer + FROM override origin + LEFT JOIN override target ON (origin.package = target.package + AND target.suite = :suite_id + AND origin.component = target.component + AND origin.type = target.type) + WHERE origin.suite = :originsuite_id + AND origin.component = :component_id + AND origin.type = :type_id""", + {'suite_id': osuite_id, 'originsuite_id': originosuite_id, + 'component_id': component_id, 'type_id': type_id}) + for i in q.fetchall(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): Logger.log(["syncing override", osuite, component, - type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""UPDATE override SET section=%s, - maintainer='%s' WHERE package='%s' AND - suite=%s AND component=%s AND type=%s""" % - (i[2], i[3], package, osuite_id, component_id, - dsc_type_id)) + session.execute("""UPDATE override + SET section = :section, + maintainer = :maintainer + WHERE package = :package AND suite = :suite_id + AND component = :component_id AND type = :type_id""", + {'section': i[2], 'maintainer': i[3], + 'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': dsc_type_id}) continue + # we can copy src_packages[package] = 1 Logger.log(["copying missing override", osuite, component, - type, package, "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, - osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, + :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': source_priority_id, + 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) for package, hasoverride in src_packages.items(): if not hasoverride: - dak.lib.utils.warn("%s has no override!" % package) + utils.warn("%s has no override!" % package) else: # binary override - for i in q.getresult(): + for i in q.fetchall(): package = i[0] if packages.has_key(package): packages[package] = 1 else: if blacklist.has_key(package): - dak.lib.utils.warn("%s in incoming, not touching" % package) + utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, - type, package, priorities[i[1]], sections[i[2]], i[3]]) + otype, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""DELETE FROM override WHERE package = - '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)) + session.execute("""DELETE FROM override + WHERE package = :package AND suite = :suite_id + AND component = :component_id AND type = :type_id + AND created < now() - interval '14 days'""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': type_id}) # Check whether originosuite has an override for us we can # copy if originosuite: - q = projectB.query("""SELECT origin.package, origin.priority, - origin.section, origin.maintainer, target.priority, - target.section, target.maintainer FROM override origin LEFT - JOIN override target ON (origin.package = target.package AND - target.suite=%s AND origin.component = target.component AND - origin.type = target.type) WHERE origin.suite = %s AND - origin.component = %s AND origin.type = %s""" % (osuite_id, - originosuite_id, component_id, type_id)) - for i in q.getresult(): + q = session.execute("""SELECT origin.package, origin.priority, origin.section, + origin.maintainer, target.priority, target.section, + target.maintainer + FROM override origin LEFT JOIN override target + ON (origin.package = target.package + AND target.suite = :suite_id + AND origin.component = target.component + AND origin.type = target.type) + WHERE origin.suite = :originsuite_id + AND origin.component = :component_id + AND origin.type = :type_id""", + {'suite_id': osuite_id, 'originsuite_id': originosuite_id, + 'component_id': component_id, 'type_id': type_id}) + for i in q.fetchall(): package = i[0] if not packages.has_key(package) or packages[package]: if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): Logger.log(["syncing override", osuite, component, - type, package, priorities[i[4]], sections[i[5]], + otype, package, priorities[i[4]], sections[i[5]], i[6], priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""UPDATE override SET priority=%s, section=%s, - maintainer='%s' WHERE package='%s' AND - suite=%s AND component=%s AND type=%s""" % - (i[1], i[2], i[3], package, osuite_id, - component_id, type_id)) + session.execute("""UPDATE override + SET priority = :priority_id, + section = :section_id, + maintainer = :maintainer + WHERE package = :package + AND suite = :suite_id + AND component = :component_id + AND type = :type_id""", + {'priority_id': i[1], 'section_id': i[2], + 'maintainer': i[3], 'package': package, + 'suite_id': osuite_id, 'component_id': component_id, + 'type_id': type_id}) continue # we can copy packages[package] = 1 Logger.log(["copying missing override", osuite, component, - type, package, priorities[i[1]], sections[i[2]], i[3]]) + otype, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': i[1], + 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]}) for package, hasoverride in packages.items(): if not hasoverride: - dak.lib.utils.warn("%s has no override!" % package) + utils.warn("%s has no override!" % package) - projectB.query("COMMIT WORK") + session.commit() sys.stdout.flush() ################################################################################ def main (): - global Logger, Options, projectB, sections, priorities + global Logger, Options, sections, priorities - Cnf = dak.lib.utils.get_conf() + cnf = Config() Arguments = [('h',"help","Check-Overrides::Options::Help"), ('n',"no-action", "Check-Overrides::Options::No-Action")] for i in [ "help", "no-action" ]: - if not Cnf.has_key("Check-Overrides::Options::%s" % (i)): - Cnf["Check-Overrides::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Check-Overrides::Options") + if not cnf.has_key("Check-Overrides::Options::%s" % (i)): + cnf["Check-Overrides::Options::%s" % (i)] = "" + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) + Options = cnf.subtree("Check-Overrides::Options") if Options["Help"]: usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - dak.lib.database.init(Cnf, projectB) + session = DBConn().session() # init sections, priorities: - q = projectB.query("SELECT id, section FROM section") - for i in q.getresult(): - sections[i[0]] = i[1] - q = projectB.query("SELECT id, priority FROM priority") - for i in q.getresult(): - priorities[i[0]] = i[1] + + # We need forward and reverse + sections = get_sections(session) + for name, entry in sections.items(): + sections[entry] = name + + priorities = get_priorities(session) + for name, entry in priorities.items(): + priorities[entry] = name if not Options["No-Action"]: - Logger = dak.lib.logging.Logger(Cnf, "check-overrides") + Logger = daklog.Logger("check-overrides") else: - Logger = dak.lib.logging.Logger(Cnf, "check-overrides", 1) + Logger = daklog.Logger("check-overrides", 1) + + for suite in session.query(Suite).filter(Suite.overrideprocess==True): + originosuite = None + originremark = '' - gen_blacklist(Cnf["Dir::Queue::Accepted"]) + if suite.overrideorigin is not None: + originosuite = get_suite(suite.overrideorigin, session) + if originosuite is None: + utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin)) + originosuite = originosuite.suite_name + originremark = " taking missing from %s" % originosuite - for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List(): - if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]: - continue + print "Processing %s%s..." % (suite.suite_name, originremark) - osuite = osuite.lower() + # Get a list of all suites that use the override file of 'suite.suite_name' as + # well as the suite + ocodename = suite.codename + suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()] + if suite.suite_id not in suiteids: + suiteids.append(suite.suite_id) - originosuite = None - originremark = "" - try: - originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite] - originosuite = originosuite.lower() - originremark = " taking missing from %s" % originosuite - except KeyError: - pass - - print "Processing %s%s..." % (osuite, originremark) - # Get a list of all suites that use the override file of 'osuite' - ocodename = Cnf["Suite::%s::codename" % osuite] - suites = [] - for suite in Cnf.SubTree("Suite").List(): - if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]: - suites.append(suite) - - q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \ - % ", ".join(map(repr, suites)).lower()) - - suiteids = [] - for i in q.getresult(): - suiteids.append(i[0]) - - if len(suiteids) != len(suites) or len(suiteids) < 1: - dak.lib.utils.fubar("Couldn't find id's of all suites: %s" % suites) - - for component in Cnf.SubTree("Component").List(): - if component == "mixed": - continue; # Ick + if len(suiteids) < 1: + utils.fubar("Couldn't find id's of all suites: %s" % suiteids) + + for component in session.query(Component).all(): # It is crucial for the dsc override creation based on binary # overrides that 'dsc' goes first - otypes = Cnf.ValueList("OverrideType") - otypes.remove("dsc") - otypes = ["dsc"] + otypes + component_name = component.component_name + otypes = ['dsc'] + for ot in session.query(OverrideType): + if ot.overridetype == 'dsc': + continue + otypes.append(ot.overridetype) + for otype in otypes: - print "Processing %s [%s - %s] using %s..." \ - % (osuite, component, otype, suites) + print "Processing %s [%s - %s]" \ + % (suite.suite_name, component_name, otype) sys.stdout.flush() - process(osuite, suiteids, originosuite, component, otype) + process(suite.suite_name, suiteids, originosuite, component_name, otype, session) Logger.close() @@ -349,4 +393,3 @@ def main (): if __name__ == '__main__': main() -