From: Mark Hymers Date: Sun, 9 Aug 2009 21:22:21 +0000 (+0100) Subject: convert check_overrides to new DB API X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=0c8e82cef114ecfcfee6e6a5ec1dfdfd585f5556 convert check_overrides to new DB API Signed-off-by: Mark Hymers --- diff --git a/dak/check_overrides.py b/dak/check_overrides.py index e21b849a..5cccfb6b 100755 --- a/dak/check_overrides.py +++ b/dak/check_overrides.py @@ -48,16 +48,17 @@ ################################################################################ -import pg, sys, os +import sys, os import apt_pkg -from daklib import database + +from daklib.config import Config +from daklib.dbconn import * from daklib import daklog from daklib import utils ################################################################################ Options = None -projectB = None Logger = None sections = {} priorities = {} @@ -81,57 +82,67 @@ def gen_blacklist(dir): entry = entry.split('_')[0] blacklist[entry] = 1 -def process(osuite, affected_suites, originosuite, component, type): - global Logger, Options, projectB, sections, priorities +def process(osuite, affected_suites, originosuite, component, otype, session): + global Logger, Options, sections, priorities - osuite_id = database.get_suite_id(osuite) - if osuite_id == -1: + o = get_suite(osuite, session) + if o is None: utils.fubar("Suite '%s' not recognised." % (osuite)) + osuite_id = o.suite_id + originosuite_id = None if originosuite: - originosuite_id = database.get_suite_id(originosuite) - if originosuite_id == -1: + oo = get_suite(originosuite, session) + if oo is None: utils.fubar("Suite '%s' not recognised." % (originosuite)) + originosuite_id = oo.suite_id - component_id = database.get_component_id(component) - if component_id == -1: + c = get_component(component, session) + if c is None: utils.fubar("Component '%s' not recognised." % (component)) + component_id = c.component_id - type_id = database.get_override_type_id(type) - if type_id == -1: - utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)) - dsc_type_id = database.get_override_type_id("dsc") + ot = get_override_type(otype, session) + if ot is None: + utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype)) + type_id = ot.overridetype_id + dsc_type_id = get_override_type("dsc", session).overridetype_id - source_priority_id = database.get_priority_id("source") + source_priority_id = get_priority("source", session).priority_id - if type == "deb" or type == "udeb": + if otype == "deb" or otype == "udeb": packages = {} - q = projectB.query(""" + # TODO: Fix to use placeholders (check how to with arrays) + q = session.execute(""" SELECT b.package FROM binaries b, bin_associations ba, files f, location l, component c - WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location - AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s -""" % (type, ",".join([ str(i) for i in affected_suites ]), component_id)) - for i in q.getresult(): + WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location + AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id +""" % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id}) + for i in q.fetchall(): packages[i[0]] = 0 src_packages = {} - q = projectB.query(""" + q = session.execute(""" SELECT s.source FROM source s, src_associations sa, files f, location l, component c WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location - AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s -""" % (",".join([ str(i) for i in affected_suites]), component_id)) - for i in q.getresult(): + AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id +""" % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id}) + for i in q.fetchall(): src_packages[i[0]] = 0 # ----------- # Drop unused overrides - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)) - projectB.query("BEGIN WORK") - if type == "dsc": - for i in q.getresult(): + q = session.execute("""SELECT package, priority, section, maintainer + FROM override WHERE suite = :suite_id + AND component = :component_id AND type = :type_id""", + {'suite_id': osuite_id, 'component_id': component_id, + 'type_id': type_id}) + # We're already within a transaction + if otype == "dsc": + for i in q.fetchall(): package = i[0] if src_packages.has_key(package): src_packages[package] = 1 @@ -140,71 +151,87 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, - type, package, priorities[i[1]], sections[i[2]], i[3]]) + otype, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""DELETE FROM override WHERE package = - '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)) + session.execute("""DELETE FROM override WHERE package = :package + AND suite = :suite_id AND component = :component_id + AND type = :type_id""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': type_id}) # create source overrides based on binary overrides, as source # overrides not always get created - q = projectB.query(""" SELECT package, priority, section, - maintainer FROM override WHERE suite = %s AND component = %s - """ % (osuite_id, component_id)) - for i in q.getresult(): + q = session.execute("""SELECT package, priority, section, maintainer + FROM override WHERE suite = :suite_id AND component = :component_id""", + {'suite_id': osuite_id, 'component_id': component_id}) + for i in q.fetchall(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: continue src_packages[package] = 1 Logger.log(["add missing override", osuite, component, - type, package, "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, - osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': source_priority_id, + 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) # Check whether originosuite has an override for us we can # copy if originosuite: - q = projectB.query("""SELECT origin.package, origin.priority, - origin.section, origin.maintainer, target.priority, - target.section, target.maintainer FROM override origin LEFT - JOIN override target ON (origin.package = target.package AND - target.suite=%s AND origin.component = target.component AND origin.type = - target.type) WHERE origin.suite = %s AND origin.component = %s - AND origin.type = %s""" % - (osuite_id, originosuite_id, component_id, type_id)) - for i in q.getresult(): + q = session.execute("""SELECT origin.package, origin.priority, origin.section, + origin.maintainer, target.priority, target.section, + target.maintainer + FROM override origin + LEFT JOIN override target ON (origin.package = target.package + AND target.suite = :suite_id + AND origin.component = target.component + AND origin.type = target.type) + WHERE origin.suite = :originsuite_id + AND origin.component = :component_id + AND origin.type = :type_id""", + {'suite_id': osuite_id, 'originsuite_id': originosuite_id, + 'component_id': component_id, 'type_id': type_id}) + for i in q.fetchall(): package = i[0] if not src_packages.has_key(package) or src_packages[package]: if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): Logger.log(["syncing override", osuite, component, - type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""UPDATE override SET section=%s, - maintainer='%s' WHERE package='%s' AND - suite=%s AND component=%s AND type=%s""" % - (i[2], i[3], package, osuite_id, component_id, - dsc_type_id)) + session.execute("""UPDATE override + SET section = :section, + maintainer = :maintainer + WHERE package = :package AND suite = :suite_id + AND component = :component_id AND type = :type_id""", + {'section': i[2], 'maintainer': i[3], + 'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': dsc_type_id}) continue + # we can copy src_packages[package] = 1 Logger.log(["copying missing override", osuite, component, - type, package, "source", sections[i[2]], i[3]]) + otype, package, "source", sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, - osuite_id, component_id, source_priority_id, i[2], - dsc_type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, + :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': source_priority_id, + 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]}) for package, hasoverride in src_packages.items(): if not hasoverride: utils.warn("%s has no override!" % package) else: # binary override - for i in q.getresult(): + for i in q.fetchall(): package = i[0] if packages.has_key(package): packages[package] = 1 @@ -213,92 +240,112 @@ SELECT s.source FROM source s, src_associations sa, files f, location l, utils.warn("%s in incoming, not touching" % package) continue Logger.log(["removing unused override", osuite, component, - type, package, priorities[i[1]], sections[i[2]], i[3]]) + otype, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""DELETE FROM override WHERE package = - '%s' AND suite = %s AND component = %s AND type = - %s""" % (package, osuite_id, component_id, type_id)) + session.execute("""DELETE FROM override + WHERE package = :package AND suite = :suite_id + AND component = :component_id AND type = :type_id""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'type_id': type_id}) # Check whether originosuite has an override for us we can # copy if originosuite: - q = projectB.query("""SELECT origin.package, origin.priority, - origin.section, origin.maintainer, target.priority, - target.section, target.maintainer FROM override origin LEFT - JOIN override target ON (origin.package = target.package AND - target.suite=%s AND origin.component = target.component AND - origin.type = target.type) WHERE origin.suite = %s AND - origin.component = %s AND origin.type = %s""" % (osuite_id, - originosuite_id, component_id, type_id)) - for i in q.getresult(): + q = session.execute("""SELECT origin.package, origin.priority, origin.section, + origin.maintainer, target.priority, target.section, + target.maintainer + FROM override origin LEFT JOIN override target + ON (origin.package = target.package + AND target.suite = :suite_id + AND origin.component = target.component + AND origin.type = target.type) + WHERE origin.suite = :originsuite_id + AND origin.component = :component_id + AND origin.type = :type_id""", + {'suite_id': osuite_id, 'originsuite_id': originosuite_id, + 'component_id': component_id, 'type_id': type_id}) + for i in q.fetchall(): package = i[0] if not packages.has_key(package) or packages[package]: if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): Logger.log(["syncing override", osuite, component, - type, package, priorities[i[4]], sections[i[5]], + otype, package, priorities[i[4]], sections[i[5]], i[6], priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""UPDATE override SET priority=%s, section=%s, - maintainer='%s' WHERE package='%s' AND - suite=%s AND component=%s AND type=%s""" % - (i[1], i[2], i[3], package, osuite_id, - component_id, type_id)) + session.execute("""UPDATE override + SET priority = :priority_id, + section = :section_id, + maintainer = :maintainer + WHERE package = :package + AND suite = :suite_id + AND component = :component_id + AND type = :type_id""", + {'priority_id': i[1], 'section_id': i[2], + 'maintainer': i[3], 'package': package, + 'suite_id': osuite_id, 'component_id': component_id, + 'type_id': type_id}) continue # we can copy packages[package] = 1 Logger.log(["copying missing override", osuite, component, type, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("""INSERT INTO override (package, suite, - component, priority, section, type, maintainer) VALUES - ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])) + session.execute("""INSERT INTO override (package, suite, component, + priority, section, type, maintainer) + VALUES (:package, :suite_id, :component_id, + :priority_id, :section_id, :type_id, :maintainer)""", + {'package': package, 'suite_id': osuite_id, + 'component_id': component_id, 'priority_id': i[1], + 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]}) for package, hasoverride in packages.items(): if not hasoverride: utils.warn("%s has no override!" % package) - projectB.query("COMMIT WORK") + session.commit() sys.stdout.flush() ################################################################################ def main (): - global Logger, Options, projectB, sections, priorities + global Logger, Options, sections, priorities - Cnf = utils.get_conf() + cnf = Config() Arguments = [('h',"help","Check-Overrides::Options::Help"), ('n',"no-action", "Check-Overrides::Options::No-Action")] for i in [ "help", "no-action" ]: - if not Cnf.has_key("Check-Overrides::Options::%s" % (i)): - Cnf["Check-Overrides::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Check-Overrides::Options") + if not cnf.has_key("Check-Overrides::Options::%s" % (i)): + cnf["Check-Overrides::Options::%s" % (i)] = "" + apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) + Options = cnf.SubTree("Check-Overrides::Options") if Options["Help"]: usage() - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - database.init(Cnf, projectB) + session = DBConn().session() # init sections, priorities: - q = projectB.query("SELECT id, section FROM section") - for i in q.getresult(): - sections[i[0]] = i[1] - q = projectB.query("SELECT id, priority FROM priority") - for i in q.getresult(): - priorities[i[0]] = i[1] + + # We need forward and reverse + sections = get_sections(session) + for name, entry in sections.items(): + sections[entry] = name + + priorities = get_priorities(session) + for name, entry in priorities.items(): + priorities[entry] = name if not Options["No-Action"]: - Logger = daklog.Logger(Cnf, "check-overrides") + Logger = daklog.Logger(cnf, "check-overrides") else: - Logger = daklog.Logger(Cnf, "check-overrides", 1) + Logger = daklog.Logger(cnf, "check-overrides", 1) - gen_blacklist(Cnf["Dir::Queue::Accepted"]) + gen_blacklist(cnf["Dir::Queue::Accepted"]) - for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List(): - if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]: + for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List(): + if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]: continue osuite = osuite.lower() @@ -306,7 +353,7 @@ def main (): originosuite = None originremark = "" try: - originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite] + originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite] originosuite = originosuite.lower() originremark = " taking missing from %s" % originosuite except KeyError: @@ -314,33 +361,30 @@ def main (): print "Processing %s%s..." % (osuite, originremark) # Get a list of all suites that use the override file of 'osuite' - ocodename = Cnf["Suite::%s::codename" % osuite].lower() + ocodename = cnf["Suite::%s::codename" % osuite].lower() suites = [] - for suite in Cnf.SubTree("Suite").List(): - if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite].lower(): - suites.append(suite) - - q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \ - % ", ".join([ repr(i) for i in suites ]).lower()) - suiteids = [] - for i in q.getresult(): - suiteids.append(i[0]) + for suite in cnf.SubTree("Suite").List(): + if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower(): + suites.append(suite) + s = get_suite(suite.lower(), session) + if s is not None: + suiteids.append(s.suite_id) if len(suiteids) != len(suites) or len(suiteids) < 1: utils.fubar("Couldn't find id's of all suites: %s" % suites) - for component in Cnf.SubTree("Component").List(): + for component in cnf.SubTree("Component").List(): # It is crucial for the dsc override creation based on binary # overrides that 'dsc' goes first - otypes = Cnf.ValueList("OverrideType") + otypes = cnf.ValueList("OverrideType") otypes.remove("dsc") otypes = ["dsc"] + otypes for otype in otypes: print "Processing %s [%s - %s] using %s..." \ % (osuite, component, otype, suites) sys.stdout.flush() - process(osuite, suiteids, originosuite, component, otype) + process(osuite, suiteids, originosuite, component, otype, session) Logger.close()