dak/check_overrides.py

   1 #!/usr/bin/env python
   2
   3 """ Cruft checker and hole filler for overrides
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
   7 @opyright: 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
   8 @copyright: 2011  Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 ######################################################################
  30 # NB: dak check-overrides is not a good idea with New Incoming as it #
  31 # doesn't take into account accepted.  You can minimize the impact   #
  32 # of this by running it immediately after dak process-accepted but   #
  33 # that's still racy because 'dak process-new' doesn't lock with 'dak #
  34 # process-accepted'.  A better long term fix is the evil plan for    #
  35 # accepted to be in the DB.                                          #
  36 ######################################################################
  37
  38 # dak check-overrides should now work fine being done during
  39 # cron.daily, for example just before 'dak make-overrides' (after 'dak
  40 # process-accepted' and 'dak make-suite-file-list'). At that point,
  41 # queue/accepted should be empty and installed, so... dak
  42 # check-overrides does now take into account suites sharing overrides
  43
  44 # TODO:
  45 # * Only update out-of-sync overrides when corresponding versions are equal to
  46 #   some degree
  47 # * consistency checks like:
  48 #   - section=debian-installer only for udeb and # dsc
  49 #   - priority=source iff dsc
  50 #   - (suite, package, 'dsc') is unique,
  51 #   - just as (suite, package, (u)deb) (yes, across components!)
  52 #   - sections match their component (each component has an own set of sections,
  53 #     could probably be reduced...)
  54
  55 ################################################################################
  56
  57 import sys, os
  58 import apt_pkg
  59
  60 from daklib.config import Config
  61 from daklib.dbconn import *
  62 from daklib import daklog
  63 from daklib import utils
  64
  65 ################################################################################
  66
  67 Options = None                 #: Commandline arguments parsed into this
  68 Logger = None                  #: Our logging object
  69 sections = {}
  70 priorities = {}
  71 blacklist = {}
  72
  73 ################################################################################
  74
  75 def usage (exit_code=0):
  76     print """Usage: dak check-overrides
  77 Check for cruft in overrides.
  78
  79   -n, --no-action            don't do anything
  80   -h, --help                 show this help and exit"""
  81
  82     sys.exit(exit_code)
  83
  84 ################################################################################
  85
  86 def process(osuite, affected_suites, originosuite, component, otype, session):
  87     global Logger, Options, sections, priorities
  88
  89     o = get_suite(osuite, session)
  90     if o is None:
  91         utils.fubar("Suite '%s' not recognised." % (osuite))
  92     osuite_id = o.suite_id
  93
  94     originosuite_id = None
  95     if originosuite:
  96         oo = get_suite(originosuite, session)
  97         if oo is None:
  98             utils.fubar("Suite '%s' not recognised." % (originosuite))
  99         originosuite_id = oo.suite_id
 100
 101     c = get_component(component, session)
 102     if c is None:
 103         utils.fubar("Component '%s' not recognised." % (component))
 104     component_id = c.component_id
 105
 106     ot = get_override_type(otype, session)
 107     if ot is None:
 108         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
 109     type_id = ot.overridetype_id
 110     dsc_type_id = get_override_type("dsc", session).overridetype_id
 111
 112     source_priority_id = get_priority("source", session).priority_id
 113
 114     if otype == "deb" or otype == "udeb":
 115         packages = {}
 116         # TODO: Fix to use placeholders (check how to with arrays)
 117         q = session.execute("""
 118 SELECT b.package
 119   FROM binaries b
 120   JOIN bin_associations ba ON b.id = ba.bin
 121   JOIN suite ON ba.suite = suite.id
 122   JOIN files_archive_map af ON b.file = af.file_id AND suite.archive_id = af.archive_id
 123  WHERE b.type = :otype AND ba.suite IN (%s) AND af.component_id = :component_id
 124 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
 125         for i in q.fetchall():
 126             packages[i[0]] = 0
 127
 128     src_packages = {}
 129     q = session.execute("""
 130 SELECT s.source FROM source s
 131   JOIN src_associations sa ON s.id = sa.source
 132   JOIN suite ON sa.suite = suite.id
 133   JOIN files_archive_map af ON s.file = af.file_id AND suite.archive_id = af.archive_id
 134  WHERE sa.suite IN (%s) AND af.component_id = :component_id
 135 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
 136     for i in q.fetchall():
 137         src_packages[i[0]] = 0
 138
 139     # -----------
 140     # Drop unused overrides
 141
 142     q = session.execute("""SELECT package, priority, section, maintainer
 143                              FROM override WHERE suite = :suite_id
 144                               AND component = :component_id AND type = :type_id""",
 145                         {'suite_id': osuite_id, 'component_id': component_id,
 146                          'type_id': type_id})
 147     # We're already within a transaction
 148     if otype == "dsc":
 149         for i in q.fetchall():
 150             package = i[0]
 151             if src_packages.has_key(package):
 152                 src_packages[package] = 1
 153             else:
 154                 if blacklist.has_key(package):
 155                     utils.warn("%s in incoming, not touching" % package)
 156                     continue
 157                 Logger.log(["removing unused override", osuite, component,
 158                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 159                 if not Options["No-Action"]:
 160                     session.execute("""DELETE FROM override WHERE package = :package
 161                                           AND suite = :suite_id AND component = :component_id
 162                                           AND type = :type_id
 163                                           AND created < now() - interval '14 days'""",
 164                                     {'package': package, 'suite_id': osuite_id,
 165                                      'component_id': component_id, 'type_id': type_id})
 166         # create source overrides based on binary overrides, as source
 167         # overrides not always get created
 168         q = session.execute("""SELECT package, priority, section, maintainer
 169                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
 170                             {'suite_id': osuite_id, 'component_id': component_id})
 171         for i in q.fetchall():
 172             package = i[0]
 173             if not src_packages.has_key(package) or src_packages[package]:
 174                 continue
 175             src_packages[package] = 1
 176
 177             Logger.log(["add missing override", osuite, component,
 178                 otype, package, "source", sections[i[2]], i[3]])
 179             if not Options["No-Action"]:
 180                 session.execute("""INSERT INTO override (package, suite, component,
 181                                                         priority, section, type, maintainer)
 182                                          VALUES (:package, :suite_id, :component_id,
 183                                                  :priority_id, :section_id, :type_id, :maintainer)""",
 184                                {'package': package, 'suite_id': osuite_id,
 185                                 'component_id': component_id, 'priority_id': source_priority_id,
 186                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 187         # Check whether originosuite has an override for us we can
 188         # copy
 189         if originosuite:
 190             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 191                                          origin.maintainer, target.priority, target.section,
 192                                          target.maintainer
 193                                     FROM override origin
 194                                LEFT JOIN override target ON (origin.package = target.package
 195                                                              AND target.suite = :suite_id
 196                                                              AND origin.component = target.component
 197                                                              AND origin.type = target.type)
 198                                    WHERE origin.suite = :originsuite_id
 199                                      AND origin.component = :component_id
 200                                      AND origin.type = :type_id""",
 201                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 202                                  'component_id': component_id, 'type_id': type_id})
 203             for i in q.fetchall():
 204                 package = i[0]
 205                 if not src_packages.has_key(package) or src_packages[package]:
 206                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 207                         Logger.log(["syncing override", osuite, component,
 208                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
 209                         if not Options["No-Action"]:
 210                             session.execute("""UPDATE override
 211                                                  SET section = :section,
 212                                                      maintainer = :maintainer
 213                                                WHERE package = :package AND suite = :suite_id
 214                                                  AND component = :component_id AND type = :type_id""",
 215                                             {'section': i[2], 'maintainer': i[3],
 216                                              'package': package, 'suite_id': osuite_id,
 217                                              'component_id': component_id, 'type_id': dsc_type_id})
 218                     continue
 219
 220                 # we can copy
 221                 src_packages[package] = 1
 222                 Logger.log(["copying missing override", osuite, component,
 223                     otype, package, "source", sections[i[2]], i[3]])
 224                 if not Options["No-Action"]:
 225                     session.execute("""INSERT INTO override (package, suite, component,
 226                                                              priority, section, type, maintainer)
 227                                             VALUES (:package, :suite_id, :component_id,
 228                                                     :priority_id, :section_id, :type_id,
 229                                                     :maintainer)""",
 230                                     {'package': package, 'suite_id': osuite_id,
 231                                      'component_id': component_id, 'priority_id': source_priority_id,
 232                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 233
 234         for package, hasoverride in src_packages.items():
 235             if not hasoverride:
 236                 utils.warn("%s has no override!" % package)
 237
 238     else: # binary override
 239         for i in q.fetchall():
 240             package = i[0]
 241             if packages.has_key(package):
 242                 packages[package] = 1
 243             else:
 244                 if blacklist.has_key(package):
 245                     utils.warn("%s in incoming, not touching" % package)
 246                     continue
 247                 Logger.log(["removing unused override", osuite, component,
 248                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 249                 if not Options["No-Action"]:
 250                     session.execute("""DELETE FROM override
 251                                         WHERE package = :package AND suite = :suite_id
 252                                           AND component = :component_id AND type = :type_id
 253                                           AND created < now() - interval '14 days'""",
 254                                     {'package': package, 'suite_id': osuite_id,
 255                                      'component_id': component_id, 'type_id': type_id})
 256
 257         # Check whether originosuite has an override for us we can
 258         # copy
 259         if originosuite:
 260             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 261                                           origin.maintainer, target.priority, target.section,
 262                                           target.maintainer
 263                                      FROM override origin LEFT JOIN override target
 264                                                           ON (origin.package = target.package
 265                                                               AND target.suite = :suite_id
 266                                                               AND origin.component = target.component
 267                                                               AND origin.type = target.type)
 268                                     WHERE origin.suite = :originsuite_id
 269                                       AND origin.component = :component_id
 270                                       AND origin.type = :type_id""",
 271                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 272                                   'component_id': component_id, 'type_id': type_id})
 273             for i in q.fetchall():
 274                 package = i[0]
 275                 if not packages.has_key(package) or packages[package]:
 276                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 277                         Logger.log(["syncing override", osuite, component,
 278                             otype, package, priorities[i[4]], sections[i[5]],
 279                             i[6], priorities[i[1]], sections[i[2]], i[3]])
 280                         if not Options["No-Action"]:
 281                             session.execute("""UPDATE override
 282                                                   SET priority = :priority_id,
 283                                                       section = :section_id,
 284                                                       maintainer = :maintainer
 285                                                 WHERE package = :package
 286                                                   AND suite = :suite_id
 287                                                   AND component = :component_id
 288                                                   AND type = :type_id""",
 289                                             {'priority_id': i[1], 'section_id': i[2],
 290                                              'maintainer': i[3], 'package': package,
 291                                              'suite_id': osuite_id, 'component_id': component_id,
 292                                              'type_id': type_id})
 293                     continue
 294                 # we can copy
 295                 packages[package] = 1
 296                 Logger.log(["copying missing override", osuite, component,
 297                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 298                 if not Options["No-Action"]:
 299                     session.execute("""INSERT INTO override (package, suite, component,
 300                                                              priority, section, type, maintainer)
 301                                             VALUES (:package, :suite_id, :component_id,
 302                                                     :priority_id, :section_id, :type_id, :maintainer)""",
 303                                     {'package': package, 'suite_id': osuite_id,
 304                                      'component_id': component_id, 'priority_id': i[1],
 305                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
 306
 307         for package, hasoverride in packages.items():
 308             if not hasoverride:
 309                 utils.warn("%s has no override!" % package)
 310
 311     session.commit()
 312     sys.stdout.flush()
 313
 314
 315 ################################################################################
 316
 317 def main ():
 318     global Logger, Options, sections, priorities
 319
 320     cnf = Config()
 321
 322     Arguments = [('h',"help","Check-Overrides::Options::Help"),
 323                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
 324     for i in [ "help", "no-action" ]:
 325         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
 326             cnf["Check-Overrides::Options::%s" % (i)] = ""
 327     apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
 328     Options = cnf.subtree("Check-Overrides::Options")
 329
 330     if Options["Help"]:
 331         usage()
 332
 333     session = DBConn().session()
 334
 335     # init sections, priorities:
 336
 337     # We need forward and reverse
 338     sections = get_sections(session)
 339     for name, entry in sections.items():
 340         sections[entry] = name
 341
 342     priorities = get_priorities(session)
 343     for name, entry in priorities.items():
 344         priorities[entry] = name
 345
 346     if not Options["No-Action"]:
 347         Logger = daklog.Logger("check-overrides")
 348     else:
 349         Logger = daklog.Logger("check-overrides", 1)
 350
 351     for suite in session.query(Suite).filter(Suite.overrideprocess==True):
 352         originosuite = None
 353         originremark = ''
 354
 355         if suite.overrideorigin is not None:
 356             originosuite = get_suite(suite.overrideorigin, session)
 357             if originosuite is None:
 358                 utils.fubar("%s has an override origin suite of %s but it doesn't exist!" % (suite.suite_name, suite.overrideorigin))
 359             originosuite = originosuite.suite_name
 360             originremark = " taking missing from %s" % originosuite
 361
 362         print "Processing %s%s..." % (suite.suite_name, originremark)
 363
 364         # Get a list of all suites that use the override file of 'suite.suite_name' as
 365         # well as the suite
 366         ocodename = suite.codename
 367         suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()]
 368         if suite.suite_id not in suiteids:
 369             suiteids.append(suite.suite_id)
 370
 371         if len(suiteids) < 1:
 372             utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
 373
 374         for component in session.query(Component).all():
 375             # It is crucial for the dsc override creation based on binary
 376             # overrides that 'dsc' goes first
 377             component_name = component.component_name
 378             otypes = ['dsc']
 379             for ot in session.query(OverrideType):
 380                 if ot.overridetype == 'dsc':
 381                     continue
 382                 otypes.append(ot.overridetype)
 383
 384             for otype in otypes:
 385                 print "Processing %s [%s - %s]" \
 386                     % (suite.suite_name, component_name, otype)
 387                 sys.stdout.flush()
 388                 process(suite.suite_name, suiteids, originosuite, component_name, otype, session)
 389
 390     Logger.close()
 391
 392 ################################################################################
 393
 394 if __name__ == '__main__':
 395     main()