X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcheck_overrides.py;h=7ed38147787a0efc2216df5dfa53715224444aa6;hb=3f4d9093048af1545f78aef3b5236a1e8094fa89;hp=b95997f78f9ea1319705e03adea1f9a76cd0356f;hpb=59fd5aa2a8be3b76dbc968429c457f096adfa472;p=dak.git diff --git a/dak/check_overrides.py b/dak/check_overrides.py index b95997f7..7ed38147 100755 --- a/dak/check_overrides.py +++ b/dak/check_overrides.py @@ -1,9 +1,8 @@ #!/usr/bin/env python # Cruft checker and hole filler for overrides -# Copyright (C) 2000, 2001, 2002, 2004 James Troup +# Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup # Copyright (C) 2005 Jeroen van Wolffelaar -# $Id: cindy,v 1.14 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,17 +21,19 @@ ################################################################################ ###################################################################### -# NB: cindy is not a good idea with New Incoming as she doesn't take # -# into account accepted. You can minimize the impact of this by # -# running her immediately after kelly but that's still racy because # -# lisa doesn't lock with kelly. A better long term fix is the evil # -# plan for accepted to be in the DB. # +# NB: dak check-overrides is not a good idea with New Incoming as it # +# doesn't take into account accepted. You can minimize the impact # +# of this by running it immediately after dak process-accepted but # +# that's still racy because 'dak process-new' doesn't lock with 'dak # +# process-accepted'. A better long term fix is the evil plan for # +# accepted to be in the DB. # ###################################################################### -# cindy should now work fine being done during cron.daily, for example just -# before denise (after kelly and jenna). At that point, queue/accepted should -# be empty and installed, so... Cindy does now take into account suites -# sharing overrides +# dak check-overrides should now work fine being done during +# cron.daily, for example just before 'dak make-overrides' (after 'dak +# process-accepted' and 'dak make-suite-file-list'). At that point, +# queue/accepted should be empty and installed, so... dak +# check-overrides does now take into account suites sharing overrides # TODO: # * Only update out-of-sync overrides when corresponding versions are equal to @@ -47,14 +48,16 @@ ################################################################################ -import pg, sys, os; -import utils, db_access, logging; -import apt_pkg; +import pg, sys, os +import apt_pkg +import daklib.database +import daklib.logging +import daklib.utils ################################################################################ -Options = None; -projectB = None; +Options = None +projectB = None Logger = None sections = {} priorities = {} @@ -63,7 +66,7 @@ blacklist = {} ################################################################################ def usage (exit_code=0): - print """Usage: cindy + print """Usage: dak check-overrides Check for cruft in overrides. -n, --no-action don't do anything @@ -79,75 +82,75 @@ def gen_blacklist(dir): blacklist[entry] = 1 def process(osuite, affected_suites, originosuite, component, type): - global Logger, Options, projectB, sections, priorities; + global Logger, Options, projectB, sections, priorities - osuite_id = db_access.get_suite_id(osuite); + osuite_id = daklib.database.get_suite_id(osuite) if osuite_id == -1: - utils.fubar("Suite '%s' not recognised." % (osuite)); + daklib.utils.fubar("Suite '%s' not recognised." % (osuite)) originosuite_id = None if originosuite: - originosuite_id = db_access.get_suite_id(originosuite); + originosuite_id = daklib.database.get_suite_id(originosuite) if originosuite_id == -1: - utils.fubar("Suite '%s' not recognised." % (originosuite)); + daklib.utils.fubar("Suite '%s' not recognised." % (originosuite)) - component_id = db_access.get_component_id(component); + component_id = daklib.database.get_component_id(component) if component_id == -1: - utils.fubar("Component '%s' not recognised." % (component)); + daklib.utils.fubar("Component '%s' not recognised." % (component)) - type_id = db_access.get_override_type_id(type); + type_id = daklib.database.get_override_type_id(type) if type_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)); - dsc_type_id = db_access.get_override_type_id("dsc"); - deb_type_id = db_access.get_override_type_id("deb") + daklib.utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)) + dsc_type_id = daklib.database.get_override_type_id("dsc") + deb_type_id = daklib.database.get_override_type_id("deb") - source_priority_id = db_access.get_priority_id("source") + source_priority_id = daklib.database.get_priority_id("source") if type == "deb" or type == "udeb": - packages = {}; + packages = {} q = projectB.query(""" SELECT b.package FROM binaries b, bin_associations ba, files f, location l, component c WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s -""" % (type, ",".join(map(str,affected_suites)), component_id)); +""" % (type, ",".join([ str(i) for i in affected_suites ]), component_id)) for i in q.getresult(): - packages[i[0]] = 0; + packages[i[0]] = 0 - src_packages = {}; + src_packages = {} q = projectB.query(""" SELECT s.source FROM source s, src_associations sa, files f, location l, component c WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s -""" % (",".join(map(str,affected_suites)), component_id)); +""" % (",".join([ str(i) for i in affected_suites]), component_id)) for i in q.getresult(): - src_packages[i[0]] = 0; + src_packages[i[0]] = 0 # ----------- # Drop unused overrides - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)); - projectB.query("BEGIN WORK"); + q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)) + projectB.query("BEGIN WORK") if type == "dsc": for i in q.getresult(): - package = i[0]; + package = i[0] if src_packages.has_key(package): src_packages[package] = 1 else: if blacklist.has_key(package): - utils.warn("%s in incoming, not touching" % package) + daklib.utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, type, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: projectB.query("""DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)); + %s""" % (package, osuite_id, component_id, type_id)) # create source overrides based on binary overrides, as source # overrides not always get created q = projectB.query(""" SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s - """ % (osuite_id, component_id)); + """ % (osuite_id, component_id)) for i in q.getresult(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: @@ -161,7 +164,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])); + dsc_type_id, i[3])) # Check whether originosuite has an override for us we can # copy if originosuite: @@ -172,7 +175,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, target.suite=%s AND origin.component = target.component AND origin.type = target.type) WHERE origin.suite = %s AND origin.component = %s AND origin.type = %s""" % - (osuite_id, originosuite_id, component_id, type_id)); + (osuite_id, originosuite_id, component_id, type_id)) for i in q.getresult(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: @@ -184,7 +187,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, maintainer='%s' WHERE package='%s' AND suite=%s AND component=%s AND type=%s""" % (i[2], i[3], package, osuite_id, component_id, - dsc_type_id)); + dsc_type_id)) continue # we can copy src_packages[package] = 1 @@ -195,27 +198,27 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])); + dsc_type_id, i[3])) for package, hasoverride in src_packages.items(): if not hasoverride: - utils.warn("%s has no override!" % package) + daklib.utils.warn("%s has no override!" % package) else: # binary override for i in q.getresult(): - package = i[0]; + package = i[0] if packages.has_key(package): packages[package] = 1 else: if blacklist.has_key(package): - utils.warn("%s in incoming, not touching" % package) + daklib.utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, type, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: projectB.query("""DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)); + %s""" % (package, osuite_id, component_id, type_id)) # Check whether originosuite has an override for us we can # copy @@ -227,7 +230,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, target.suite=%s AND origin.component = target.component AND origin.type = target.type) WHERE origin.suite = %s AND origin.component = %s AND origin.type = %s""" % (osuite_id, - originosuite_id, component_id, type_id)); + originosuite_id, component_id, type_id)) for i in q.getresult(): package = i[0] if not packages.has_key(package) or packages[package]: @@ -240,7 +243,7 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, maintainer='%s' WHERE package='%s' AND suite=%s AND component=%s AND type=%s""" % (i[1], i[2], i[3], package, osuite_id, - component_id, type_id)); + component_id, type_id)) continue # we can copy packages[package] = 1 @@ -249,36 +252,36 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, if not Options["No-Action"]: projectB.query("""INSERT INTO override (package, suite, component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])); + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])) for package, hasoverride in packages.items(): if not hasoverride: - utils.warn("%s has no override!" % package) + daklib.utils.warn("%s has no override!" % package) - projectB.query("COMMIT WORK"); + projectB.query("COMMIT WORK") sys.stdout.flush() ################################################################################ def main (): - global Logger, Options, projectB, sections, priorities; + global Logger, Options, projectB, sections, priorities - Cnf = utils.get_conf() + Cnf = daklib.utils.get_conf() - Arguments = [('h',"help","Cindy::Options::Help"), - ('n',"no-action", "Cindy::Options::No-Action")]; + Arguments = [('h',"help","Check-Overrides::Options::Help"), + ('n',"no-action", "Check-Overrides::Options::No-Action")] for i in [ "help", "no-action" ]: - if not Cnf.has_key("Cindy::Options::%s" % (i)): - Cnf["Cindy::Options::%s" % (i)] = ""; - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); - Options = Cnf.SubTree("Cindy::Options") + if not Cnf.has_key("Check-Overrides::Options::%s" % (i)): + Cnf["Check-Overrides::Options::%s" % (i)] = "" + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) + Options = Cnf.SubTree("Check-Overrides::Options") if Options["Help"]: - usage(); + usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); - db_access.init(Cnf, projectB); + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + daklib.database.init(Cnf, projectB) # init sections, priorities: q = projectB.query("SELECT id, section FROM section") @@ -289,14 +292,14 @@ def main (): priorities[i[0]] = i[1] if not Options["No-Action"]: - Logger = logging.Logger(Cnf, "cindy") + Logger = daklib.logging.Logger(Cnf, "check-overrides") else: - Logger = logging.Logger(Cnf, "cindy", 1) + Logger = daklib.logging.Logger(Cnf, "check-overrides", 1) gen_blacklist(Cnf["Dir::Queue::Accepted"]) - for osuite in Cnf.SubTree("Cindy::OverrideSuites").List(): - if "1" != Cnf["Cindy::OverrideSuites::%s::Process" % osuite]: + for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List(): + if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]: continue osuite = osuite.lower() @@ -304,13 +307,13 @@ def main (): originosuite = None originremark = "" try: - originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite]; + originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite] originosuite = originosuite.lower() originremark = " taking missing from %s" % originosuite except KeyError: pass - print "Processing %s%s..." % (osuite, originremark); + print "Processing %s%s..." % (osuite, originremark) # Get a list of all suites that use the override file of 'osuite' ocodename = Cnf["Suite::%s::codename" % osuite] suites = [] @@ -319,14 +322,14 @@ def main (): suites.append(suite) q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \ - % ", ".join(map(repr, suites)).lower()) + % ", ".join([ repr(i) for i in suites ]).lower()) suiteids = [] for i in q.getresult(): suiteids.append(i[0]) if len(suiteids) != len(suites) or len(suiteids) < 1: - utils.fubar("Couldn't find id's of all suites: %s" % suites) + daklib.utils.fubar("Couldn't find id's of all suites: %s" % suites) for component in Cnf.SubTree("Component").List(): if component == "mixed": @@ -338,9 +341,9 @@ def main (): otypes = ["dsc"] + otypes for otype in otypes: print "Processing %s [%s - %s] using %s..." \ - % (osuite, component, otype, suites); + % (osuite, component, otype, suites) sys.stdout.flush() - process(osuite, suiteids, originosuite, component, otype); + process(osuite, suiteids, originosuite, component, otype) Logger.close()