dak/check_overrides.py

   1 #!/usr/bin/env python
   2
   3 """ Cruft checker and hole filler for overrides """
   4 # Copyright (C) 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
   5 # Copyright (C) 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
   6
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 ################################################################################
  22
  23 ######################################################################
  24 # NB: dak check-overrides is not a good idea with New Incoming as it #
  25 # doesn't take into account accepted.  You can minimize the impact   #
  26 # of this by running it immediately after dak process-accepted but   #
  27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
  28 # process-accepted'.  A better long term fix is the evil plan for    #
  29 # accepted to be in the DB.                                          #
  30 ######################################################################
  31
  32 # dak check-overrides should now work fine being done during
  33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
  34 # process-accepted' and 'dak make-suite-file-list'). At that point,
  35 # queue/accepted should be empty and installed, so... dak
  36 # check-overrides does now take into account suites sharing overrides
  37
  38 # TODO:
  39 # * Only update out-of-sync overrides when corresponding versions are equal to
  40 #   some degree
  41 # * consistency checks like:
  42 #   - section=debian-installer only for udeb and # dsc
  43 #   - priority=source iff dsc
  44 #   - (suite, package, 'dsc') is unique,
  45 #   - just as (suite, package, (u)deb) (yes, across components!)
  46 #   - sections match their component (each component has an own set of sections,
  47 #     could probably be reduced...)
  48
  49 ################################################################################
  50
  51 import sys, os
  52 import apt_pkg
  53
  54 from daklib.config import Config
  55 from daklib.dbconn import *
  56 from daklib import daklog
  57 from daklib import utils
  58
  59 ################################################################################
  60
  61 Options = None
  62 Logger = None
  63 sections = {}
  64 priorities = {}
  65 blacklist = {}
  66
  67 ################################################################################
  68
  69 def usage (exit_code=0):
  70     print """Usage: dak check-overrides
  71 Check for cruft in overrides.
  72
  73   -n, --no-action            don't do anything
  74   -h, --help                 show this help and exit"""
  75
  76     sys.exit(exit_code)
  77
  78 ################################################################################
  79
  80 def gen_blacklist(dir):
  81     for entry in os.listdir(dir):
  82         entry = entry.split('_')[0]
  83         blacklist[entry] = 1
  84
  85 def process(osuite, affected_suites, originosuite, component, otype, session):
  86     global Logger, Options, sections, priorities
  87
  88     o = get_suite(osuite, session)
  89     if o is None:
  90         utils.fubar("Suite '%s' not recognised." % (osuite))
  91     osuite_id = o.suite_id
  92
  93     originosuite_id = None
  94     if originosuite:
  95         oo = get_suite(originosuite, session)
  96         if oo is None:
  97             utils.fubar("Suite '%s' not recognised." % (originosuite))
  98         originosuite_id = oo.suite_id
  99
 100     c = get_component(component, session)
 101     if c is None:
 102         utils.fubar("Component '%s' not recognised." % (component))
 103     component_id = c.component_id
 104
 105     ot = get_override_type(otype, session)
 106     if ot is None:
 107         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
 108     type_id = ot.overridetype_id
 109     dsc_type_id = get_override_type("dsc", session).overridetype_id
 110
 111     source_priority_id = get_priority("source", session).priority_id
 112
 113     if otype == "deb" or otype == "udeb":
 114         packages = {}
 115         # TODO: Fix to use placeholders (check how to with arrays)
 116         q = session.execute("""
 117 SELECT b.package FROM binaries b, bin_associations ba, files f,
 118                               location l, component c
 119  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
 120    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
 121 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
 122         for i in q.fetchall():
 123             packages[i[0]] = 0
 124
 125     src_packages = {}
 126     q = session.execute("""
 127 SELECT s.source FROM source s, src_associations sa, files f, location l,
 128                      component c
 129  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
 130    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
 131 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
 132     for i in q.fetchall():
 133         src_packages[i[0]] = 0
 134
 135     # -----------
 136     # Drop unused overrides
 137
 138     q = session.execute("""SELECT package, priority, section, maintainer
 139                              FROM override WHERE suite = :suite_id
 140                               AND component = :component_id AND type = :type_id""",
 141                         {'suite_id': osuite_id, 'component_id': component_id,
 142                          'type_id': type_id})
 143     # We're already within a transaction
 144     if otype == "dsc":
 145         for i in q.fetchall():
 146             package = i[0]
 147             if src_packages.has_key(package):
 148                 src_packages[package] = 1
 149             else:
 150                 if blacklist.has_key(package):
 151                     utils.warn("%s in incoming, not touching" % package)
 152                     continue
 153                 Logger.log(["removing unused override", osuite, component,
 154                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 155                 if not Options["No-Action"]:
 156                     session.execute("""DELETE FROM override WHERE package = :package
 157                                           AND suite = :suite_id AND component = :component_id
 158                                           AND type = :type_id""",
 159                                     {'package': package, 'suite_id': osuite_id,
 160                                      'component_id': component_id, 'type_id': type_id})
 161         # create source overrides based on binary overrides, as source
 162         # overrides not always get created
 163         q = session.execute("""SELECT package, priority, section, maintainer
 164                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
 165                             {'suite_id': osuite_id, 'component_id': component_id})
 166         for i in q.fetchall():
 167             package = i[0]
 168             if not src_packages.has_key(package) or src_packages[package]:
 169                 continue
 170             src_packages[package] = 1
 171
 172             Logger.log(["add missing override", osuite, component,
 173                 otype, package, "source", sections[i[2]], i[3]])
 174             if not Options["No-Action"]:
 175                 session.execute("""INSERT INTO override (package, suite, component,
 176                                                         priority, section, type, maintainer)
 177                                          VALUES (:package, :suite_id, :component_id,
 178                                                  :priority_id, :section_id, :type_id, :maintainer)""",
 179                                {'package': package, 'suite_id': osuite_id,
 180                                 'component_id': component_id, 'priority_id': source_priority_id,
 181                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 182         # Check whether originosuite has an override for us we can
 183         # copy
 184         if originosuite:
 185             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 186                                          origin.maintainer, target.priority, target.section,
 187                                          target.maintainer
 188                                     FROM override origin
 189                                LEFT JOIN override target ON (origin.package = target.package
 190                                                              AND target.suite = :suite_id
 191                                                              AND origin.component = target.component
 192                                                              AND origin.type = target.type)
 193                                    WHERE origin.suite = :originsuite_id
 194                                      AND origin.component = :component_id
 195                                      AND origin.type = :type_id""",
 196                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 197                                  'component_id': component_id, 'type_id': type_id})
 198             for i in q.fetchall():
 199                 package = i[0]
 200                 if not src_packages.has_key(package) or src_packages[package]:
 201                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 202                         Logger.log(["syncing override", osuite, component,
 203                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
 204                         if not Options["No-Action"]:
 205                             session.execute("""UPDATE override
 206                                                  SET section = :section,
 207                                                      maintainer = :maintainer
 208                                                WHERE package = :package AND suite = :suite_id
 209                                                  AND component = :component_id AND type = :type_id""",
 210                                             {'section': i[2], 'maintainer': i[3],
 211                                              'package': package, 'suite_id': osuite_id,
 212                                              'component_id': component_id, 'type_id': dsc_type_id})
 213                     continue
 214
 215                 # we can copy
 216                 src_packages[package] = 1
 217                 Logger.log(["copying missing override", osuite, component,
 218                     otype, package, "source", sections[i[2]], i[3]])
 219                 if not Options["No-Action"]:
 220                     session.execute("""INSERT INTO override (package, suite, component,
 221                                                              priority, section, type, maintainer)
 222                                             VALUES (:package, :suite_id, :component_id,
 223                                                     :priority_id, :section_id, :type_id,
 224                                                     :maintainer)""",
 225                                     {'package': package, 'suite_id': osuite_id,
 226                                      'component_id': component_id, 'priority_id': source_priority_id,
 227                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 228
 229         for package, hasoverride in src_packages.items():
 230             if not hasoverride:
 231                 utils.warn("%s has no override!" % package)
 232
 233     else: # binary override
 234         for i in q.fetchall():
 235             package = i[0]
 236             if packages.has_key(package):
 237                 packages[package] = 1
 238             else:
 239                 if blacklist.has_key(package):
 240                     utils.warn("%s in incoming, not touching" % package)
 241                     continue
 242                 Logger.log(["removing unused override", osuite, component,
 243                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 244                 if not Options["No-Action"]:
 245                     session.execute("""DELETE FROM override
 246                                         WHERE package = :package AND suite = :suite_id
 247                                           AND component = :component_id AND type = :type_id""",
 248                                     {'package': package, 'suite_id': osuite_id,
 249                                      'component_id': component_id, 'type_id': type_id})
 250
 251         # Check whether originosuite has an override for us we can
 252         # copy
 253         if originosuite:
 254             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 255                                           origin.maintainer, target.priority, target.section,
 256                                           target.maintainer
 257                                      FROM override origin LEFT JOIN override target
 258                                                           ON (origin.package = target.package
 259                                                               AND target.suite = :suite_id
 260                                                               AND origin.component = target.component
 261                                                               AND origin.type = target.type)
 262                                     WHERE origin.suite = :originsuite_id
 263                                       AND origin.component = :component_id
 264                                       AND origin.type = :type_id""",
 265                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 266                                   'component_id': component_id, 'type_id': type_id})
 267             for i in q.fetchall():
 268                 package = i[0]
 269                 if not packages.has_key(package) or packages[package]:
 270                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 271                         Logger.log(["syncing override", osuite, component,
 272                             otype, package, priorities[i[4]], sections[i[5]],
 273                             i[6], priorities[i[1]], sections[i[2]], i[3]])
 274                         if not Options["No-Action"]:
 275                             session.execute("""UPDATE override
 276                                                   SET priority = :priority_id,
 277                                                       section = :section_id,
 278                                                       maintainer = :maintainer
 279                                                 WHERE package = :package
 280                                                   AND suite = :suite_id
 281                                                   AND component = :component_id
 282                                                   AND type = :type_id""",
 283                                             {'priority_id': i[1], 'section_id': i[2],
 284                                              'maintainer': i[3], 'package': package,
 285                                              'suite_id': osuite_id, 'component_id': component_id,
 286                                              'type_id': type_id})
 287                     continue
 288                 # we can copy
 289                 packages[package] = 1
 290                 Logger.log(["copying missing override", osuite, component,
 291                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 292                 if not Options["No-Action"]:
 293                     session.execute("""INSERT INTO override (package, suite, component,
 294                                                              priority, section, type, maintainer)
 295                                             VALUES (:package, :suite_id, :component_id,
 296                                                     :priority_id, :section_id, :type_id, :maintainer)""",
 297                                     {'package': package, 'suite_id': osuite_id,
 298                                      'component_id': component_id, 'priority_id': i[1],
 299                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
 300
 301         for package, hasoverride in packages.items():
 302             if not hasoverride:
 303                 utils.warn("%s has no override!" % package)
 304
 305     session.commit()
 306     sys.stdout.flush()
 307
 308
 309 ################################################################################
 310
 311 def main ():
 312     global Logger, Options, sections, priorities
 313
 314     cnf = Config()
 315
 316     Arguments = [('h',"help","Check-Overrides::Options::Help"),
 317                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
 318     for i in [ "help", "no-action" ]:
 319         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
 320             cnf["Check-Overrides::Options::%s" % (i)] = ""
 321     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 322     Options = cnf.SubTree("Check-Overrides::Options")
 323
 324     if Options["Help"]:
 325         usage()
 326
 327     session = DBConn().session()
 328
 329     # init sections, priorities:
 330
 331     # We need forward and reverse
 332     sections = get_sections(session)
 333     for name, entry in sections.items():
 334         sections[entry] = name
 335
 336     priorities = get_priorities(session)
 337     for name, entry in priorities.items():
 338         priorities[entry] = name
 339
 340     if not Options["No-Action"]:
 341         Logger = daklog.Logger(cnf, "check-overrides")
 342     else:
 343         Logger = daklog.Logger(cnf, "check-overrides", 1)
 344
 345     gen_blacklist(cnf["Dir::Queue::Accepted"])
 346
 347     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
 348         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
 349             continue
 350
 351         osuite = osuite.lower()
 352
 353         originosuite = None
 354         originremark = ""
 355         try:
 356             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
 357             originosuite = originosuite.lower()
 358             originremark = " taking missing from %s" % originosuite
 359         except KeyError:
 360             pass
 361
 362         print "Processing %s%s..." % (osuite, originremark)
 363         # Get a list of all suites that use the override file of 'osuite'
 364         ocodename = cnf["Suite::%s::codename" % osuite].lower()
 365         suites = []
 366         suiteids = []
 367         for suite in cnf.SubTree("Suite").List():
 368             if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
 369                 suites.append(suite)
 370                 s = get_suite(suite.lower(), session)
 371                 if s is not None:
 372                     suiteids.append(s.suite_id)
 373
 374         if len(suiteids) != len(suites) or len(suiteids) < 1:
 375             utils.fubar("Couldn't find id's of all suites: %s" % suites)
 376
 377         for component in cnf.SubTree("Component").List():
 378             # It is crucial for the dsc override creation based on binary
 379             # overrides that 'dsc' goes first
 380             otypes = cnf.ValueList("OverrideType")
 381             otypes.remove("dsc")
 382             otypes = ["dsc"] + otypes
 383             for otype in otypes:
 384                 print "Processing %s [%s - %s] using %s..." \
 385                     % (osuite, component, otype, suites)
 386                 sys.stdout.flush()
 387                 process(osuite, suiteids, originosuite, component, otype, session)
 388
 389     Logger.close()
 390
 391 ################################################################################
 392
 393 if __name__ == '__main__':
 394     main()