#!/usr/bin/env python
-# Cruft checker and hole filler for overrides
-# Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
-# Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
+""" Cruft checker and hole filler for overrides
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
+@opyright: 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
+@copyright: 2011 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-import pg, sys, os
+import sys, os
import apt_pkg
-from daklib import database
-from daklib import logging
+
+from daklib.config import Config
+from daklib.dbconn import *
+from daklib import daklog
from daklib import utils
################################################################################
-Options = None
-projectB = None
-Logger = None
+Options = None #: Commandline arguments parsed into this
+Logger = None #: Our logging object
sections = {}
priorities = {}
blacklist = {}
################################################################################
-def gen_blacklist(dir):
- for entry in os.listdir(dir):
- entry = entry.split('_')[0]
- blacklist[entry] = 1
-
-def process(osuite, affected_suites, originosuite, component, type):
- global Logger, Options, projectB, sections, priorities
+def process(osuite, affected_suites, originosuite, component, otype, session):
+ global Logger, Options, sections, priorities
- osuite_id = database.get_suite_id(osuite)
- if osuite_id == -1:
+ o = get_suite(osuite, session)
+ if o is None:
utils.fubar("Suite '%s' not recognised." % (osuite))
+ osuite_id = o.suite_id
+
originosuite_id = None
if originosuite:
- originosuite_id = database.get_suite_id(originosuite)
- if originosuite_id == -1:
+ oo = get_suite(originosuite, session)
+ if oo is None:
utils.fubar("Suite '%s' not recognised." % (originosuite))
+ originosuite_id = oo.suite_id
- component_id = database.get_component_id(component)
- if component_id == -1:
+ c = get_component(component, session)
+ if c is None:
utils.fubar("Component '%s' not recognised." % (component))
+ component_id = c.component_id
- type_id = database.get_override_type_id(type)
- if type_id == -1:
- utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
- dsc_type_id = database.get_override_type_id("dsc")
+ ot = get_override_type(otype, session)
+ if ot is None:
+ utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
+ type_id = ot.overridetype_id
+ dsc_type_id = get_override_type("dsc", session).overridetype_id
- source_priority_id = database.get_priority_id("source")
+ source_priority_id = get_priority("source", session).priority_id
- if type == "deb" or type == "udeb":
+ if otype == "deb" or otype == "udeb":
packages = {}
- q = projectB.query("""
-SELECT b.package FROM binaries b, bin_associations ba, files f,
- location l, component c
- WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
- AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
-""" % (type, ",".join([ str(i) for i in affected_suites ]), component_id))
- for i in q.getresult():
+ # TODO: Fix to use placeholders (check how to with arrays)
+ q = session.execute("""
+SELECT b.package
+ FROM binaries b
+ JOIN bin_associations ba ON b.id = ba.bin
+ JOIN suite ON ba.suite = suite.id
+ JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
+ WHERE b.type = :otype AND ba.suite IN (%s) AND af.component_id = :component_id
+""" % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
+ for i in q.fetchall():
packages[i[0]] = 0
src_packages = {}
- q = projectB.query("""
-SELECT s.source FROM source s, src_associations sa, files f, location l,
- component c
- WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
- AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
-""" % (",".join([ str(i) for i in affected_suites]), component_id))
- for i in q.getresult():
+ q = session.execute("""
+SELECT s.source FROM source s
+ JOIN src_associations sa ON s.id = sa.source
+ JOIN suite ON sa.suite = suite.id
+ JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
+ WHERE sa.suite IN (%s) AND af.component_id = :component_id
+""" % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
+ for i in q.fetchall():
src_packages[i[0]] = 0
# -----------
# Drop unused overrides
- q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
- projectB.query("BEGIN WORK")
- if type == "dsc":
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
+ # We're already within a transaction
+ if otype == "dsc":
+ for i in q.fetchall():
package = i[0]
if src_packages.has_key(package):
src_packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override WHERE package = :package
+ AND suite = :suite_id AND component = :component_id
+ AND type = :type_id
+ AND created < now() - interval '14 days'""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# create source overrides based on binary overrides, as source
# overrides not always get created
- q = projectB.query(""" SELECT package, priority, section,
- maintainer FROM override WHERE suite = %s AND component = %s
- """ % (osuite_id, component_id))
- for i in q.getresult():
+ q = session.execute("""SELECT package, priority, section, maintainer
+ FROM override WHERE suite = :suite_id AND component = :component_id""",
+ {'suite_id': osuite_id, 'component_id': component_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
continue
src_packages[package] = 1
Logger.log(["add missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND origin.type =
- target.type) WHERE origin.suite = %s AND origin.component = %s
- AND origin.type = %s""" %
- (osuite_id, originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin
+ LEFT JOIN override target ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not src_packages.has_key(package) or src_packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[2], i[3], package, osuite_id, component_id,
- dsc_type_id))
+ session.execute("""UPDATE override
+ SET section = :section,
+ maintainer = :maintainer
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id""",
+ {'section': i[2], 'maintainer': i[3],
+ 'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': dsc_type_id})
continue
+
# we can copy
src_packages[package] = 1
Logger.log(["copying missing override", osuite, component,
- type, package, "source", sections[i[2]], i[3]])
+ otype, package, "source", sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
- osuite_id, component_id, source_priority_id, i[2],
- dsc_type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id,
+ :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': source_priority_id,
+ 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
for package, hasoverride in src_packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
else: # binary override
- for i in q.getresult():
+ for i in q.fetchall():
package = i[0]
if packages.has_key(package):
packages[package] = 1
utils.warn("%s in incoming, not touching" % package)
continue
Logger.log(["removing unused override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""DELETE FROM override WHERE package =
- '%s' AND suite = %s AND component = %s AND type =
- %s""" % (package, osuite_id, component_id, type_id))
+ session.execute("""DELETE FROM override
+ WHERE package = :package AND suite = :suite_id
+ AND component = :component_id AND type = :type_id
+ AND created < now() - interval '14 days'""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'type_id': type_id})
# Check whether originosuite has an override for us we can
# copy
if originosuite:
- q = projectB.query("""SELECT origin.package, origin.priority,
- origin.section, origin.maintainer, target.priority,
- target.section, target.maintainer FROM override origin LEFT
- JOIN override target ON (origin.package = target.package AND
- target.suite=%s AND origin.component = target.component AND
- origin.type = target.type) WHERE origin.suite = %s AND
- origin.component = %s AND origin.type = %s""" % (osuite_id,
- originosuite_id, component_id, type_id))
- for i in q.getresult():
+ q = session.execute("""SELECT origin.package, origin.priority, origin.section,
+ origin.maintainer, target.priority, target.section,
+ target.maintainer
+ FROM override origin LEFT JOIN override target
+ ON (origin.package = target.package
+ AND target.suite = :suite_id
+ AND origin.component = target.component
+ AND origin.type = target.type)
+ WHERE origin.suite = :originsuite_id
+ AND origin.component = :component_id
+ AND origin.type = :type_id""",
+ {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
+ 'component_id': component_id, 'type_id': type_id})
+ for i in q.fetchall():
package = i[0]
if not packages.has_key(package) or packages[package]:
if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
Logger.log(["syncing override", osuite, component,
- type, package, priorities[i[4]], sections[i[5]],
+ otype, package, priorities[i[4]], sections[i[5]],
i[6], priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""UPDATE override SET priority=%s, section=%s,
- maintainer='%s' WHERE package='%s' AND
- suite=%s AND component=%s AND type=%s""" %
- (i[1], i[2], i[3], package, osuite_id,
- component_id, type_id))
+ session.execute("""UPDATE override
+ SET priority = :priority_id,
+ section = :section_id,
+ maintainer = :maintainer
+ WHERE package = :package
+ AND suite = :suite_id
+ AND component = :component_id
+ AND type = :type_id""",
+ {'priority_id': i[1], 'section_id': i[2],
+ 'maintainer': i[3], 'package': package,
+ 'suite_id': osuite_id, 'component_id': component_id,
+ 'type_id': type_id})
continue
# we can copy
packages[package] = 1
Logger.log(["copying missing override", osuite, component,
- type, package, priorities[i[1]], sections[i[2]], i[3]])
+ otype, package, priorities[i[1]], sections[i[2]], i[3]])
if not Options["No-Action"]:
- projectB.query("""INSERT INTO override (package, suite,
- component, priority, section, type, maintainer) VALUES
- ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
+ session.execute("""INSERT INTO override (package, suite, component,
+ priority, section, type, maintainer)
+ VALUES (:package, :suite_id, :component_id,
+ :priority_id, :section_id, :type_id, :maintainer)""",
+ {'package': package, 'suite_id': osuite_id,
+ 'component_id': component_id, 'priority_id': i[1],
+ 'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
for package, hasoverride in packages.items():
if not hasoverride:
utils.warn("%s has no override!" % package)
- projectB.query("COMMIT WORK")
+ session.commit()
sys.stdout.flush()
################################################################################
def main ():
- global Logger, Options, projectB, sections, priorities
+ global Logger, Options, sections, priorities
- Cnf = utils.get_conf()
+ cnf = Config()
Arguments = [('h',"help","Check-Overrides::Options::Help"),
('n',"no-action", "Check-Overrides::Options::No-Action")]
for i in [ "help", "no-action" ]:
- if not Cnf.has_key("Check-Overrides::Options::%s" % (i)):
- Cnf["Check-Overrides::Options::%s" % (i)] = ""
- apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
- Options = Cnf.SubTree("Check-Overrides::Options")
+ if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
+ cnf["Check-Overrides::Options::%s" % (i)] = ""
+ apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.subtree("Check-Overrides::Options")
if Options["Help"]:
usage()
- projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
- database.init(Cnf, projectB)
+ session = DBConn().session()
# init sections, priorities:
- q = projectB.query("SELECT id, section FROM section")
- for i in q.getresult():
- sections[i[0]] = i[1]
- q = projectB.query("SELECT id, priority FROM priority")
- for i in q.getresult():
- priorities[i[0]] = i[1]
+
+ # We need forward and reverse
+ sections = get_sections(session)
+ for name, entry in sections.items():
+ sections[entry] = name
+
+ priorities = get_priorities(session)
+ for name, entry in priorities.items():
+ priorities[entry] = name
if not Options["No-Action"]:
- Logger = logging.Logger(Cnf, "check-overrides")
+ Logger = daklog.Logger("check-overrides")
else:
- Logger = logging.Logger(Cnf, "check-overrides", 1)
+ Logger = daklog.Logger("check-overrides", 1)
- gen_blacklist(Cnf["Dir::Queue::Accepted"])
+ for suite in session.query(Suite).filter(Suite.overrideprocess==True):
+ originosuite = None
+ originremark = ''
- for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
- if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
- continue
+ if suite.overrideorigin is not None:
+ originosuite = get_suite(suite.overrideorigin, session)
+ if originosuite is None:
+ utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin))
+ originosuite = originosuite.suite_name
+ originremark = " taking missing from %s" % originosuite
- osuite = osuite.lower()
+ print "Processing %s%s..." % (suite.suite_name, originremark)
- originosuite = None
- originremark = ""
- try:
- originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
- originosuite = originosuite.lower()
- originremark = " taking missing from %s" % originosuite
- except KeyError:
- pass
-
- print "Processing %s%s..." % (osuite, originremark)
- # Get a list of all suites that use the override file of 'osuite'
- ocodename = Cnf["Suite::%s::codename" % osuite].lower()
- suites = []
- for suite in Cnf.SubTree("Suite").List():
- if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite].lower():
- suites.append(suite)
-
- q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
- % ", ".join([ repr(i) for i in suites ]).lower())
-
- suiteids = []
- for i in q.getresult():
- suiteids.append(i[0])
-
- if len(suiteids) != len(suites) or len(suiteids) < 1:
- utils.fubar("Couldn't find id's of all suites: %s" % suites)
-
- for component in Cnf.SubTree("Component").List():
- if component == "mixed":
- continue; # Ick
+ # Get a list of all suites that use the override file of 'suite.suite_name' as
+ # well as the suite
+ ocodename = suite.codename
+ suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()]
+ if suite.suite_id not in suiteids:
+ suiteids.append(suite.suite_id)
+
+ if len(suiteids) < 1:
+ utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
+
+ for component in session.query(Component).all():
# It is crucial for the dsc override creation based on binary
# overrides that 'dsc' goes first
- otypes = Cnf.ValueList("OverrideType")
- otypes.remove("dsc")
- otypes = ["dsc"] + otypes
+ component_name = component.component_name
+ otypes = ['dsc']
+ for ot in session.query(OverrideType):
+ if ot.overridetype == 'dsc':
+ continue
+ otypes.append(ot.overridetype)
+
for otype in otypes:
- print "Processing %s [%s - %s] using %s..." \
- % (osuite, component, otype, suites)
+ print "Processing %s [%s - %s]" \
+ % (suite.suite_name, component_name, otype)
sys.stdout.flush()
- process(osuite, suiteids, originosuite, component, otype)
+ process(suite.suite_name, suiteids, originosuite, component_name, otype, session)
Logger.close()
projects / dak.git / blobdiff