X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=cindy;h=b95997f78f9ea1319705e03adea1f9a76cd0356f;hb=9540d873fa78598454af57f5f8a4875969ed0439;hp=af4370380b677125ec3c80d46dd9e0955b070319;hpb=4b41dc06229eed89f52f397fc7df5d665fa092cf;p=dak.git diff --git a/cindy b/cindy index af437038..b95997f7 100755 --- a/cindy +++ b/cindy @@ -1,8 +1,9 @@ #!/usr/bin/env python -# Output override files for apt-ftparchive and indices/ -# Copyright (C) 2000 James Troup -# $Id: cindy,v 1.1 2001-01-16 21:52:37 troup Exp $ +# Cruft checker and hole filler for overrides +# Copyright (C) 2000, 2001, 2002, 2004 James Troup +# Copyright (C) 2005 Jeroen van Wolffelaar +# $Id: cindy,v 1.14 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,106 +19,332 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +################################################################################ + +###################################################################### +# NB: cindy is not a good idea with New Incoming as she doesn't take # +# into account accepted. You can minimize the impact of this by # +# running her immediately after kelly but that's still racy because # +# lisa doesn't lock with kelly. A better long term fix is the evil # +# plan for accepted to be in the DB. # +###################################################################### + +# cindy should now work fine being done during cron.daily, for example just +# before denise (after kelly and jenna). At that point, queue/accepted should +# be empty and installed, so... Cindy does now take into account suites +# sharing overrides -# X-Listening-To: Sanitarium / Master of the Puppets - Metallica +# TODO: +# * Only update out-of-sync overrides when corresponding versions are equal to +# some degree +# * consistency checks like: +# - section=debian-installer only for udeb and # dsc +# - priority=source iff dsc +# - (suite, package, 'dsc') is unique, +# - just as (suite, package, (u)deb) (yes, across components!) +# - sections match their component (each component has an own set of sections, +# could probably be reduced...) ################################################################################ -import pg, sys, string -import utils, db_access, natalie +import pg, sys, os; +import utils, db_access, logging; import apt_pkg; ################################################################################ -Cnf = None; +Options = None; projectB = None; -override = {} +Logger = None +sections = {} +priorities = {} +blacklist = {} ################################################################################ -def process(suite, component, type): - global override; - - suite_id = db_access.get_suite_id(suite); - if suite_id == -1: - sys.stderr.write("Suite '%s' not recognised.\n" % (suite)); - sys.exit(2); +def usage (exit_code=0): + print """Usage: cindy +Check for cruft in overrides. + + -n, --no-action don't do anything + -h, --help show this help and exit""" + + sys.exit(exit_code) + +################################################################################ + +def gen_blacklist(dir): + for entry in os.listdir(dir): + entry = entry.split('_')[0] + blacklist[entry] = 1 + +def process(osuite, affected_suites, originosuite, component, type): + global Logger, Options, projectB, sections, priorities; + + osuite_id = db_access.get_suite_id(osuite); + if osuite_id == -1: + utils.fubar("Suite '%s' not recognised." % (osuite)); + originosuite_id = None + if originosuite: + originosuite_id = db_access.get_suite_id(originosuite); + if originosuite_id == -1: + utils.fubar("Suite '%s' not recognised." % (originosuite)); component_id = db_access.get_component_id(component); if component_id == -1: - sys.stderr.write("Component '%s' not recognised.\n" % (component)); - sys.exit(2); + utils.fubar("Component '%s' not recognised." % (component)); type_id = db_access.get_override_type_id(type); if type_id == -1: - sys.stderr.write("Type '%s' not recognised. (Valid types are deb, udeb and dsc.)\n" % (type)); - sys.exit(2); + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)); dsc_type_id = db_access.get_override_type_id("dsc"); + deb_type_id = db_access.get_override_type_id("deb") + + source_priority_id = db_access.get_priority_id("source") if type == "deb" or type == "udeb": packages = {}; - q = projectB.query("SELECT DISTINCT b.package FROM binaries b, bin_associations ba WHERE b.id = ba.bin AND ba.suite = %s" % (suite_id)); + q = projectB.query(""" +SELECT b.package FROM binaries b, bin_associations ba, files f, + location l, component c + WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location + AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s +""" % (type, ",".join(map(str,affected_suites)), component_id)); for i in q.getresult(): - packages[i[0]] = ""; + packages[i[0]] = 0; src_packages = {}; - q = projectB.query("SELECT DISTINCT s.source FROM source s, src_associations sa WHERE s.id = sa.source AND sa.suite = %s" % (suite_id)); + q = projectB.query(""" +SELECT s.source FROM source s, src_associations sa, files f, location l, + component c + WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location + AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s +""" % (",".join(map(str,affected_suites)), component_id)); for i in q.getresult(): - src_packages[i[0]] = ""; + src_packages[i[0]] = 0; - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (suite_id, component_id, type_id)); + # ----------- + # Drop unused overrides + + q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)); projectB.query("BEGIN WORK"); - for i in q.getresult(): - package = i[0]; - if type == "deb" or type == "udeb": - if not packages.has_key(package): - if not src_packages.has_key(package): - print "DELETING: %s" % (package); - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" - % (package, suite_id, component_id, type_id)); - else: - print "MAKING SOURCE: %s" % (package); - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" - % (package, suite_id, component_id, type_id)); - # Then if source doesn't already have a copy, insert it into source - q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (package, suite_id, component_id, dsc_type_id)); - if q.getresult() == []: - projectB.query("INSERT INTO override (package, suite, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')" % (package, suite_id, component_id, i[1], i[2], dsc_type_id, i[3])); - else: # dsc - if not src_packages.has_key(package): - print "DELETING: %s" % (package); - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" - % (package, suite_id, component_id, type_id)); - projectB.query("COMMIT WORK"); + if type == "dsc": + for i in q.getresult(): + package = i[0]; + if src_packages.has_key(package): + src_packages[package] = 1 + else: + if blacklist.has_key(package): + utils.warn("%s in incoming, not touching" % package) + continue + Logger.log(["removing unused override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""DELETE FROM override WHERE package = + '%s' AND suite = %s AND component = %s AND type = + %s""" % (package, osuite_id, component_id, type_id)); + # create source overrides based on binary overrides, as source + # overrides not always get created + q = projectB.query(""" SELECT package, priority, section, + maintainer FROM override WHERE suite = %s AND component = %s + """ % (osuite_id, component_id)); + for i in q.getresult(): + package = i[0] + if not src_packages.has_key(package) or src_packages[package]: + continue + src_packages[package] = 1 + Logger.log(["add missing override", osuite, component, + type, package, "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, + osuite_id, component_id, source_priority_id, i[2], + dsc_type_id, i[3])); + # Check whether originosuite has an override for us we can + # copy + if originosuite: + q = projectB.query("""SELECT origin.package, origin.priority, + origin.section, origin.maintainer, target.priority, + target.section, target.maintainer FROM override origin LEFT + JOIN override target ON (origin.package = target.package AND + target.suite=%s AND origin.component = target.component AND origin.type = + target.type) WHERE origin.suite = %s AND origin.component = %s + AND origin.type = %s""" % + (osuite_id, originosuite_id, component_id, type_id)); + for i in q.getresult(): + package = i[0] + if not src_packages.has_key(package) or src_packages[package]: + if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): + Logger.log(["syncing override", osuite, component, + type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""UPDATE override SET section=%s, + maintainer='%s' WHERE package='%s' AND + suite=%s AND component=%s AND type=%s""" % + (i[2], i[3], package, osuite_id, component_id, + dsc_type_id)); + continue + # we can copy + src_packages[package] = 1 + Logger.log(["copying missing override", osuite, component, + type, package, "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, + osuite_id, component_id, source_priority_id, i[2], + dsc_type_id, i[3])); + + for package, hasoverride in src_packages.items(): + if not hasoverride: + utils.warn("%s has no override!" % package) + + else: # binary override + for i in q.getresult(): + package = i[0]; + if packages.has_key(package): + packages[package] = 1 + else: + if blacklist.has_key(package): + utils.warn("%s in incoming, not touching" % package) + continue + Logger.log(["removing unused override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""DELETE FROM override WHERE package = + '%s' AND suite = %s AND component = %s AND type = + %s""" % (package, osuite_id, component_id, type_id)); + + # Check whether originosuite has an override for us we can + # copy + if originosuite: + q = projectB.query("""SELECT origin.package, origin.priority, + origin.section, origin.maintainer, target.priority, + target.section, target.maintainer FROM override origin LEFT + JOIN override target ON (origin.package = target.package AND + target.suite=%s AND origin.component = target.component AND + origin.type = target.type) WHERE origin.suite = %s AND + origin.component = %s AND origin.type = %s""" % (osuite_id, + originosuite_id, component_id, type_id)); + for i in q.getresult(): + package = i[0] + if not packages.has_key(package) or packages[package]: + if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): + Logger.log(["syncing override", osuite, component, + type, package, priorities[i[4]], sections[i[5]], + i[6], priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""UPDATE override SET priority=%s, section=%s, + maintainer='%s' WHERE package='%s' AND + suite=%s AND component=%s AND type=%s""" % + (i[1], i[2], i[3], package, osuite_id, + component_id, type_id)); + continue + # we can copy + packages[package] = 1 + Logger.log(["copying missing override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])); + + for package, hasoverride in packages.items(): + if not hasoverride: + utils.warn("%s has no override!" % package) + + projectB.query("COMMIT WORK"); + sys.stdout.flush() + ################################################################################ def main (): - global Cnf, projectB, override; - - apt_pkg.init(); - - Cnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); - Arguments = [('D',"debug","Denise::Options::Debug", "IntVal"), - ('h',"help","Denise::Options::Help"), - ('V',"version","Denise::Options::Version")]; - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - - projectB = pg.connect('projectb', 'localhost'); + global Logger, Options, projectB, sections, priorities; + + Cnf = utils.get_conf() + + Arguments = [('h',"help","Cindy::Options::Help"), + ('n',"no-action", "Cindy::Options::No-Action")]; + for i in [ "help", "no-action" ]: + if not Cnf.has_key("Cindy::Options::%s" % (i)): + Cnf["Cindy::Options::%s" % (i)] = ""; + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + Options = Cnf.SubTree("Cindy::Options") + + if Options["Help"]: + usage(); + + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init(Cnf, projectB); - for suite in [ "stable", "unstable" ]: - sys.stderr.write("Processing %s...\n" % (suite)); + # init sections, priorities: + q = projectB.query("SELECT id, section FROM section") + for i in q.getresult(): + sections[i[0]] = i[1] + q = projectB.query("SELECT id, priority FROM priority") + for i in q.getresult(): + priorities[i[0]] = i[1] + + if not Options["No-Action"]: + Logger = logging.Logger(Cnf, "cindy") + else: + Logger = logging.Logger(Cnf, "cindy", 1) + + gen_blacklist(Cnf["Dir::Queue::Accepted"]) + + for osuite in Cnf.SubTree("Cindy::OverrideSuites").List(): + if "1" != Cnf["Cindy::OverrideSuites::%s::Process" % osuite]: + continue + + osuite = osuite.lower() + + originosuite = None + originremark = "" + try: + originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite]; + originosuite = originosuite.lower() + originremark = " taking missing from %s" % originosuite + except KeyError: + pass + + print "Processing %s%s..." % (osuite, originremark); + # Get a list of all suites that use the override file of 'osuite' + ocodename = Cnf["Suite::%s::codename" % osuite] + suites = [] + for suite in Cnf.SubTree("Suite").List(): + if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]: + suites.append(suite) + + q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \ + % ", ".join(map(repr, suites)).lower()) + + suiteids = [] + for i in q.getresult(): + suiteids.append(i[0]) + + if len(suiteids) != len(suites) or len(suiteids) < 1: + utils.fubar("Couldn't find id's of all suites: %s" % suites) + for component in Cnf.SubTree("Component").List(): if component == "mixed": continue; # Ick - for type in Cnf.SubTree("OverrideType").List(): - print "Processing %s [%s - %s]..." % (suite, component, type); - process(suite, component, type); + # It is crucial for the dsc override creation based on binary + # overrides that 'dsc' goes first + otypes = Cnf.ValueList("OverrideType") + otypes.remove("dsc") + otypes = ["dsc"] + otypes + for otype in otypes: + print "Processing %s [%s - %s] using %s..." \ + % (osuite, component, otype, suites); + sys.stdout.flush() + process(osuite, suiteids, originosuite, component, otype); -####################################################################################### + Logger.close() + +################################################################################ if __name__ == '__main__': main()