dak/check_overrides.py

   1 #!/usr/bin/env python
   2
   3 """ Cruft checker and hole filler for overrides """
   4 # Copyright (C) 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
   5 # Copyright (C) 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 ######################################################################
  24 # NB: dak check-overrides is not a good idea with New Incoming as it #
  25 # doesn't take into account accepted.  You can minimize the impact   #
  26 # of this by running it immediately after dak process-accepted but   #
  27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
  28 # process-accepted'.  A better long term fix is the evil plan for    #
  29 # accepted to be in the DB.                                          #
  30 ######################################################################
  31
  32 # dak check-overrides should now work fine being done during
  33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
  34 # process-accepted' and 'dak make-suite-file-list'). At that point,
  35 # queue/accepted should be empty and installed, so... dak
  36 # check-overrides does now take into account suites sharing overrides
  37
  38 # TODO:
  39 # * Only update out-of-sync overrides when corresponding versions are equal to
  40 #   some degree
  41 # * consistency checks like:
  42 #   - section=debian-installer only for udeb and # dsc
  43 #   - priority=source iff dsc
  44 #   - (suite, package, 'dsc') is unique,
  45 #   - just as (suite, package, (u)deb) (yes, across components!)
  46 #   - sections match their component (each component has an own set of sections,
  47 #     could probably be reduced...)
  48
  49 ################################################################################
  50
  51 import sys, os
  52 import apt_pkg
  53
  54 from daklib.config import Config
  55 from daklib.dbconn import *
  56 from daklib import daklog
  57 from daklib import utils
  58
  59 ################################################################################
  60
  61 Options = None
  62 Logger = None
  63 sections = {}
  64 priorities = {}
  65 blacklist = {}
  66
  67 ################################################################################
  68
  69 def usage (exit_code=0):
  70     print """Usage: dak check-overrides
  71 Check for cruft in overrides.
  72
  73   -n, --no-action            don't do anything
  74   -h, --help                 show this help and exit"""
  75
  76     sys.exit(exit_code)
  77
  78 ################################################################################
  79
  80 def process(osuite, affected_suites, originosuite, component, otype, session):
  81     global Logger, Options, sections, priorities
  82
  83     o = get_suite(osuite, session)
  84     if o is None:
  85         utils.fubar("Suite '%s' not recognised." % (osuite))
  86     osuite_id = o.suite_id
  87
  88     originosuite_id = None
  89     if originosuite:
  90         oo = get_suite(originosuite, session)
  91         if oo is None:
  92             utils.fubar("Suite '%s' not recognised." % (originosuite))
  93         originosuite_id = oo.suite_id
  94
  95     c = get_component(component, session)
  96     if c is None:
  97         utils.fubar("Component '%s' not recognised." % (component))
  98     component_id = c.component_id
  99
 100     ot = get_override_type(otype, session)
 101     if ot is None:
 102         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
 103     type_id = ot.overridetype_id
 104     dsc_type_id = get_override_type("dsc", session).overridetype_id
 105
 106     source_priority_id = get_priority("source", session).priority_id
 107
 108     if otype == "deb" or otype == "udeb":
 109         packages = {}
 110         # TODO: Fix to use placeholders (check how to with arrays)
 111         q = session.execute("""
 112 SELECT b.package FROM binaries b, bin_associations ba, files f,
 113                               location l, component c
 114  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
 115    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
 116 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
 117         for i in q.fetchall():
 118             packages[i[0]] = 0
 119
 120     src_packages = {}
 121     q = session.execute("""
 122 SELECT s.source FROM source s, src_associations sa, files f, location l,
 123                      component c
 124  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
 125    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
 126 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
 127     for i in q.fetchall():
 128         src_packages[i[0]] = 0
 129
 130     # -----------
 131     # Drop unused overrides
 132
 133     q = session.execute("""SELECT package, priority, section, maintainer
 134                              FROM override WHERE suite = :suite_id
 135                               AND component = :component_id AND type = :type_id""",
 136                         {'suite_id': osuite_id, 'component_id': component_id,
 137                          'type_id': type_id})
 138     # We're already within a transaction
 139     if otype == "dsc":
 140         for i in q.fetchall():
 141             package = i[0]
 142             if src_packages.has_key(package):
 143                 src_packages[package] = 1
 144             else:
 145                 if blacklist.has_key(package):
 146                     utils.warn("%s in incoming, not touching" % package)
 147                     continue
 148                 Logger.log(["removing unused override", osuite, component,
 149                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 150                 if not Options["No-Action"]:
 151                     session.execute("""DELETE FROM override WHERE package = :package
 152                                           AND suite = :suite_id AND component = :component_id
 153                                           AND type = :type_id""",
 154                                     {'package': package, 'suite_id': osuite_id,
 155                                      'component_id': component_id, 'type_id': type_id})
 156         # create source overrides based on binary overrides, as source
 157         # overrides not always get created
 158         q = session.execute("""SELECT package, priority, section, maintainer
 159                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
 160                             {'suite_id': osuite_id, 'component_id': component_id})
 161         for i in q.fetchall():
 162             package = i[0]
 163             if not src_packages.has_key(package) or src_packages[package]:
 164                 continue
 165             src_packages[package] = 1
 166
 167             Logger.log(["add missing override", osuite, component,
 168                 otype, package, "source", sections[i[2]], i[3]])
 169             if not Options["No-Action"]:
 170                 session.execute("""INSERT INTO override (package, suite, component,
 171                                                         priority, section, type, maintainer)
 172                                          VALUES (:package, :suite_id, :component_id,
 173                                                  :priority_id, :section_id, :type_id, :maintainer)""",
 174                                {'package': package, 'suite_id': osuite_id,
 175                                 'component_id': component_id, 'priority_id': source_priority_id,
 176                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 177         # Check whether originosuite has an override for us we can
 178         # copy
 179         if originosuite:
 180             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 181                                          origin.maintainer, target.priority, target.section,
 182                                          target.maintainer
 183                                     FROM override origin
 184                                LEFT JOIN override target ON (origin.package = target.package
 185                                                              AND target.suite = :suite_id
 186                                                              AND origin.component = target.component
 187                                                              AND origin.type = target.type)
 188                                    WHERE origin.suite = :originsuite_id
 189                                      AND origin.component = :component_id
 190                                      AND origin.type = :type_id""",
 191                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 192                                  'component_id': component_id, 'type_id': type_id})
 193             for i in q.fetchall():
 194                 package = i[0]
 195                 if not src_packages.has_key(package) or src_packages[package]:
 196                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 197                         Logger.log(["syncing override", osuite, component,
 198                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
 199                         if not Options["No-Action"]:
 200                             session.execute("""UPDATE override
 201                                                  SET section = :section,
 202                                                      maintainer = :maintainer
 203                                                WHERE package = :package AND suite = :suite_id
 204                                                  AND component = :component_id AND type = :type_id""",
 205                                             {'section': i[2], 'maintainer': i[3],
 206                                              'package': package, 'suite_id': osuite_id,
 207                                              'component_id': component_id, 'type_id': dsc_type_id})
 208                     continue
 209
 210                 # we can copy
 211                 src_packages[package] = 1
 212                 Logger.log(["copying missing override", osuite, component,
 213                     otype, package, "source", sections[i[2]], i[3]])
 214                 if not Options["No-Action"]:
 215                     session.execute("""INSERT INTO override (package, suite, component,
 216                                                              priority, section, type, maintainer)
 217                                             VALUES (:package, :suite_id, :component_id,
 218                                                     :priority_id, :section_id, :type_id,
 219                                                     :maintainer)""",
 220                                     {'package': package, 'suite_id': osuite_id,
 221                                      'component_id': component_id, 'priority_id': source_priority_id,
 222                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 223
 224         for package, hasoverride in src_packages.items():
 225             if not hasoverride:
 226                 utils.warn("%s has no override!" % package)
 227
 228     else: # binary override
 229         for i in q.fetchall():
 230             package = i[0]
 231             if packages.has_key(package):
 232                 packages[package] = 1
 233             else:
 234                 if blacklist.has_key(package):
 235                     utils.warn("%s in incoming, not touching" % package)
 236                     continue
 237                 Logger.log(["removing unused override", osuite, component,
 238                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 239                 if not Options["No-Action"]:
 240                     session.execute("""DELETE FROM override
 241                                         WHERE package = :package AND suite = :suite_id
 242                                           AND component = :component_id AND type = :type_id""",
 243                                     {'package': package, 'suite_id': osuite_id,
 244                                      'component_id': component_id, 'type_id': type_id})
 245
 246         # Check whether originosuite has an override for us we can
 247         # copy
 248         if originosuite:
 249             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 250                                           origin.maintainer, target.priority, target.section,
 251                                           target.maintainer
 252                                      FROM override origin LEFT JOIN override target
 253                                                           ON (origin.package = target.package
 254                                                               AND target.suite = :suite_id
 255                                                               AND origin.component = target.component
 256                                                               AND origin.type = target.type)
 257                                     WHERE origin.suite = :originsuite_id
 258                                       AND origin.component = :component_id
 259                                       AND origin.type = :type_id""",
 260                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 261                                   'component_id': component_id, 'type_id': type_id})
 262             for i in q.fetchall():
 263                 package = i[0]
 264                 if not packages.has_key(package) or packages[package]:
 265                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 266                         Logger.log(["syncing override", osuite, component,
 267                             otype, package, priorities[i[4]], sections[i[5]],
 268                             i[6], priorities[i[1]], sections[i[2]], i[3]])
 269                         if not Options["No-Action"]:
 270                             session.execute("""UPDATE override
 271                                                   SET priority = :priority_id,
 272                                                       section = :section_id,
 273                                                       maintainer = :maintainer
 274                                                 WHERE package = :package
 275                                                   AND suite = :suite_id
 276                                                   AND component = :component_id
 277                                                   AND type = :type_id""",
 278                                             {'priority_id': i[1], 'section_id': i[2],
 279                                              'maintainer': i[3], 'package': package,
 280                                              'suite_id': osuite_id, 'component_id': component_id,
 281                                              'type_id': type_id})
 282                     continue
 283                 # we can copy
 284                 packages[package] = 1
 285                 Logger.log(["copying missing override", osuite, component,
 286                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 287                 if not Options["No-Action"]:
 288                     session.execute("""INSERT INTO override (package, suite, component,
 289                                                              priority, section, type, maintainer)
 290                                             VALUES (:package, :suite_id, :component_id,
 291                                                     :priority_id, :section_id, :type_id, :maintainer)""",
 292                                     {'package': package, 'suite_id': osuite_id,
 293                                      'component_id': component_id, 'priority_id': i[1],
 294                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
 295
 296         for package, hasoverride in packages.items():
 297             if not hasoverride:
 298                 utils.warn("%s has no override!" % package)
 299
 300     session.commit()
 301     sys.stdout.flush()
 302
 303
 304 ################################################################################
 305
 306 def main ():
 307     global Logger, Options, sections, priorities
 308
 309     cnf = Config()
 310
 311     Arguments = [('h',"help","Check-Overrides::Options::Help"),
 312                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
 313     for i in [ "help", "no-action" ]:
 314         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
 315             cnf["Check-Overrides::Options::%s" % (i)] = ""
 316     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 317     Options = cnf.SubTree("Check-Overrides::Options")
 318
 319     if Options["Help"]:
 320         usage()
 321
 322     session = DBConn().session()
 323
 324     # init sections, priorities:
 325
 326     # We need forward and reverse
 327     sections = get_sections(session)
 328     for name, entry in sections.items():
 329         sections[entry] = name
 330
 331     priorities = get_priorities(session)
 332     for name, entry in priorities.items():
 333         priorities[entry] = name
 334
 335     if not Options["No-Action"]:
 336         Logger = daklog.Logger(cnf, "check-overrides")
 337     else:
 338         Logger = daklog.Logger(cnf, "check-overrides", 1)
 339
 340     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
 341         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
 342             continue
 343
 344         osuite = osuite.lower()
 345
 346         originosuite = None
 347         originremark = ""
 348         try:
 349             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
 350             originosuite = originosuite.lower()
 351             originremark = " taking missing from %s" % originosuite
 352         except KeyError:
 353             pass
 354
 355         print "Processing %s%s..." % (osuite, originremark)
 356         # Get a list of all suites that use the override file of 'osuite'
 357         ocodename = cnf["Suite::%s::codename" % osuite].lower()
 358         suites = []
 359         suiteids = []
 360         for suite in cnf.SubTree("Suite").List():
 361             if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
 362                 suites.append(suite)
 363                 s = get_suite(suite.lower(), session)
 364                 if s is not None:
 365                     suiteids.append(s.suite_id)
 366
 367         if len(suiteids) != len(suites) or len(suiteids) < 1:
 368             utils.fubar("Couldn't find id's of all suites: %s" % suites)
 369
 370         for component in cnf.SubTree("Component").List():
 371             # It is crucial for the dsc override creation based on binary
 372             # overrides that 'dsc' goes first
 373             otypes = cnf.ValueList("OverrideType")
 374             otypes.remove("dsc")
 375             otypes = ["dsc"] + otypes
 376             for otype in otypes:
 377                 print "Processing %s [%s - %s] using %s..." \
 378                     % (osuite, component, otype, suites)
 379                 sys.stdout.flush()
 380                 process(osuite, suiteids, originosuite, component, otype, session)
 381
 382     Logger.close()
 383
 384 ################################################################################
 385
 386 if __name__ == '__main__':
 387     main()