dak/check_overrides.py

   1 #!/usr/bin/env python
   2
   3 """ Cruft checker and hole filler for overrides
   4
   5 @contact: Debian FTPMaster <ftpmaster@debian.org>
   6 @copyright: 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
   7 @opyright: 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
   8 @copyright: 2011  Joerg Jaspert <joerg@debian.org>
   9 @license: GNU General Public License version 2 or later
  10
  11 """
  12
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 2 of the License, or
  16 # (at your option) any later version.
  17
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU General Public License for more details.
  22
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, write to the Free Software
  25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26
  27 ################################################################################
  28
  29 ######################################################################
  30 # NB: dak check-overrides is not a good idea with New Incoming as it #
  31 # doesn't take into account accepted.  You can minimize the impact   #
  32 # of this by running it immediately after dak process-accepted but   #
  33 # that's still racy because 'dak process-new' doesn't lock with 'dak #
  34 # process-accepted'.  A better long term fix is the evil plan for    #
  35 # accepted to be in the DB.                                          #
  36 ######################################################################
  37
  38 # dak check-overrides should now work fine being done during
  39 # cron.daily, for example just before 'dak make-overrides' (after 'dak
  40 # process-accepted' and 'dak make-suite-file-list'). At that point,
  41 # queue/accepted should be empty and installed, so... dak
  42 # check-overrides does now take into account suites sharing overrides
  43
  44 # TODO:
  45 # * Only update out-of-sync overrides when corresponding versions are equal to
  46 #   some degree
  47 # * consistency checks like:
  48 #   - section=debian-installer only for udeb and # dsc
  49 #   - priority=source iff dsc
  50 #   - (suite, package, 'dsc') is unique,
  51 #   - just as (suite, package, (u)deb) (yes, across components!)
  52 #   - sections match their component (each component has an own set of sections,
  53 #     could probably be reduced...)
  54
  55 ################################################################################
  56
  57 import sys, os
  58 import apt_pkg
  59
  60 from daklib.config import Config
  61 from daklib.dbconn import *
  62 from daklib import daklog
  63 from daklib import utils
  64
  65 ################################################################################
  66
  67 Options = None                 #: Commandline arguments parsed into this
  68 Logger = None                  #: Our logging object
  69 sections = {}
  70 priorities = {}
  71 blacklist = {}
  72
  73 ################################################################################
  74
  75 def usage (exit_code=0):
  76     print """Usage: dak check-overrides
  77 Check for cruft in overrides.
  78
  79   -n, --no-action            don't do anything
  80   -h, --help                 show this help and exit"""
  81
  82     sys.exit(exit_code)
  83
  84 ################################################################################
  85
  86 def process(osuite, affected_suites, originosuite, component, otype, session):
  87     global Logger, Options, sections, priorities
  88
  89     o = get_suite(osuite, session)
  90     if o is None:
  91         utils.fubar("Suite '%s' not recognised." % (osuite))
  92     osuite_id = o.suite_id
  93
  94     originosuite_id = None
  95     if originosuite:
  96         oo = get_suite(originosuite, session)
  97         if oo is None:
  98             utils.fubar("Suite '%s' not recognised." % (originosuite))
  99         originosuite_id = oo.suite_id
 100
 101     c = get_component(component, session)
 102     if c is None:
 103         utils.fubar("Component '%s' not recognised." % (component))
 104     component_id = c.component_id
 105
 106     ot = get_override_type(otype, session)
 107     if ot is None:
 108         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
 109     type_id = ot.overridetype_id
 110     dsc_type_id = get_override_type("dsc", session).overridetype_id
 111
 112     source_priority_id = get_priority("source", session).priority_id
 113
 114     if otype == "deb" or otype == "udeb":
 115         packages = {}
 116         # TODO: Fix to use placeholders (check how to with arrays)
 117         q = session.execute("""
 118 SELECT b.package FROM binaries b, bin_associations ba, files f,
 119                               location l, component c
 120  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
 121    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
 122 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
 123         for i in q.fetchall():
 124             packages[i[0]] = 0
 125
 126     src_packages = {}
 127     q = session.execute("""
 128 SELECT s.source FROM source s, src_associations sa, files f, location l,
 129                      component c
 130  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
 131    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
 132 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
 133     for i in q.fetchall():
 134         src_packages[i[0]] = 0
 135
 136     # -----------
 137     # Drop unused overrides
 138
 139     q = session.execute("""SELECT package, priority, section, maintainer
 140                              FROM override WHERE suite = :suite_id
 141                               AND component = :component_id AND type = :type_id""",
 142                         {'suite_id': osuite_id, 'component_id': component_id,
 143                          'type_id': type_id})
 144     # We're already within a transaction
 145     if otype == "dsc":
 146         for i in q.fetchall():
 147             package = i[0]
 148             if src_packages.has_key(package):
 149                 src_packages[package] = 1
 150             else:
 151                 if blacklist.has_key(package):
 152                     utils.warn("%s in incoming, not touching" % package)
 153                     continue
 154                 Logger.log(["removing unused override", osuite, component,
 155                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 156                 if not Options["No-Action"]:
 157                     session.execute("""DELETE FROM override WHERE package = :package
 158                                           AND suite = :suite_id AND component = :component_id
 159                                           AND type = :type_id
 160                                           AND created < now() - interval '14 days'""",
 161                                     {'package': package, 'suite_id': osuite_id,
 162                                      'component_id': component_id, 'type_id': type_id})
 163         # create source overrides based on binary overrides, as source
 164         # overrides not always get created
 165         q = session.execute("""SELECT package, priority, section, maintainer
 166                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
 167                             {'suite_id': osuite_id, 'component_id': component_id})
 168         for i in q.fetchall():
 169             package = i[0]
 170             if not src_packages.has_key(package) or src_packages[package]:
 171                 continue
 172             src_packages[package] = 1
 173
 174             Logger.log(["add missing override", osuite, component,
 175                 otype, package, "source", sections[i[2]], i[3]])
 176             if not Options["No-Action"]:
 177                 session.execute("""INSERT INTO override (package, suite, component,
 178                                                         priority, section, type, maintainer)
 179                                          VALUES (:package, :suite_id, :component_id,
 180                                                  :priority_id, :section_id, :type_id, :maintainer)""",
 181                                {'package': package, 'suite_id': osuite_id,
 182                                 'component_id': component_id, 'priority_id': source_priority_id,
 183                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 184         # Check whether originosuite has an override for us we can
 185         # copy
 186         if originosuite:
 187             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 188                                          origin.maintainer, target.priority, target.section,
 189                                          target.maintainer
 190                                     FROM override origin
 191                                LEFT JOIN override target ON (origin.package = target.package
 192                                                              AND target.suite = :suite_id
 193                                                              AND origin.component = target.component
 194                                                              AND origin.type = target.type)
 195                                    WHERE origin.suite = :originsuite_id
 196                                      AND origin.component = :component_id
 197                                      AND origin.type = :type_id""",
 198                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 199                                  'component_id': component_id, 'type_id': type_id})
 200             for i in q.fetchall():
 201                 package = i[0]
 202                 if not src_packages.has_key(package) or src_packages[package]:
 203                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 204                         Logger.log(["syncing override", osuite, component,
 205                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
 206                         if not Options["No-Action"]:
 207                             session.execute("""UPDATE override
 208                                                  SET section = :section,
 209                                                      maintainer = :maintainer
 210                                                WHERE package = :package AND suite = :suite_id
 211                                                  AND component = :component_id AND type = :type_id""",
 212                                             {'section': i[2], 'maintainer': i[3],
 213                                              'package': package, 'suite_id': osuite_id,
 214                                              'component_id': component_id, 'type_id': dsc_type_id})
 215                     continue
 216
 217                 # we can copy
 218                 src_packages[package] = 1
 219                 Logger.log(["copying missing override", osuite, component,
 220                     otype, package, "source", sections[i[2]], i[3]])
 221                 if not Options["No-Action"]:
 222                     session.execute("""INSERT INTO override (package, suite, component,
 223                                                              priority, section, type, maintainer)
 224                                             VALUES (:package, :suite_id, :component_id,
 225                                                     :priority_id, :section_id, :type_id,
 226                                                     :maintainer)""",
 227                                     {'package': package, 'suite_id': osuite_id,
 228                                      'component_id': component_id, 'priority_id': source_priority_id,
 229                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
 230
 231         for package, hasoverride in src_packages.items():
 232             if not hasoverride:
 233                 utils.warn("%s has no override!" % package)
 234
 235     else: # binary override
 236         for i in q.fetchall():
 237             package = i[0]
 238             if packages.has_key(package):
 239                 packages[package] = 1
 240             else:
 241                 if blacklist.has_key(package):
 242                     utils.warn("%s in incoming, not touching" % package)
 243                     continue
 244                 Logger.log(["removing unused override", osuite, component,
 245                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 246                 if not Options["No-Action"]:
 247                     session.execute("""DELETE FROM override
 248                                         WHERE package = :package AND suite = :suite_id
 249                                           AND component = :component_id AND type = :type_id
 250                                           AND created < now() - interval '14 days'""",
 251                                     {'package': package, 'suite_id': osuite_id,
 252                                      'component_id': component_id, 'type_id': type_id})
 253
 254         # Check whether originosuite has an override for us we can
 255         # copy
 256         if originosuite:
 257             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
 258                                           origin.maintainer, target.priority, target.section,
 259                                           target.maintainer
 260                                      FROM override origin LEFT JOIN override target
 261                                                           ON (origin.package = target.package
 262                                                               AND target.suite = :suite_id
 263                                                               AND origin.component = target.component
 264                                                               AND origin.type = target.type)
 265                                     WHERE origin.suite = :originsuite_id
 266                                       AND origin.component = :component_id
 267                                       AND origin.type = :type_id""",
 268                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
 269                                   'component_id': component_id, 'type_id': type_id})
 270             for i in q.fetchall():
 271                 package = i[0]
 272                 if not packages.has_key(package) or packages[package]:
 273                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
 274                         Logger.log(["syncing override", osuite, component,
 275                             otype, package, priorities[i[4]], sections[i[5]],
 276                             i[6], priorities[i[1]], sections[i[2]], i[3]])
 277                         if not Options["No-Action"]:
 278                             session.execute("""UPDATE override
 279                                                   SET priority = :priority_id,
 280                                                       section = :section_id,
 281                                                       maintainer = :maintainer
 282                                                 WHERE package = :package
 283                                                   AND suite = :suite_id
 284                                                   AND component = :component_id
 285                                                   AND type = :type_id""",
 286                                             {'priority_id': i[1], 'section_id': i[2],
 287                                              'maintainer': i[3], 'package': package,
 288                                              'suite_id': osuite_id, 'component_id': component_id,
 289                                              'type_id': type_id})
 290                     continue
 291                 # we can copy
 292                 packages[package] = 1
 293                 Logger.log(["copying missing override", osuite, component,
 294                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
 295                 if not Options["No-Action"]:
 296                     session.execute("""INSERT INTO override (package, suite, component,
 297                                                              priority, section, type, maintainer)
 298                                             VALUES (:package, :suite_id, :component_id,
 299                                                     :priority_id, :section_id, :type_id, :maintainer)""",
 300                                     {'package': package, 'suite_id': osuite_id,
 301                                      'component_id': component_id, 'priority_id': i[1],
 302                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
 303
 304         for package, hasoverride in packages.items():
 305             if not hasoverride:
 306                 utils.warn("%s has no override!" % package)
 307
 308     session.commit()
 309     sys.stdout.flush()
 310
 311
 312 ################################################################################
 313
 314 def main ():
 315     global Logger, Options, sections, priorities
 316
 317     cnf = Config()
 318
 319     Arguments = [('h',"help","Check-Overrides::Options::Help"),
 320                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
 321     for i in [ "help", "no-action" ]:
 322         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
 323             cnf["Check-Overrides::Options::%s" % (i)] = ""
 324     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
 325     Options = cnf.SubTree("Check-Overrides::Options")
 326
 327     if Options["Help"]:
 328         usage()
 329
 330     session = DBConn().session()
 331
 332     # init sections, priorities:
 333
 334     # We need forward and reverse
 335     sections = get_sections(session)
 336     for name, entry in sections.items():
 337         sections[entry] = name
 338
 339     priorities = get_priorities(session)
 340     for name, entry in priorities.items():
 341         priorities[entry] = name
 342
 343     if not Options["No-Action"]:
 344         Logger = daklog.Logger(cnf, "check-overrides")
 345     else:
 346         Logger = daklog.Logger(cnf, "check-overrides", 1)
 347
 348     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
 349         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
 350             continue
 351
 352         osuite = osuite.lower()
 353
 354         originosuite = None
 355         originremark = ""
 356         try:
 357             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
 358             originosuite = originosuite.lower()
 359             originremark = " taking missing from %s" % originosuite
 360         except KeyError:
 361             pass
 362
 363         print "Processing %s%s..." % (osuite, originremark)
 364         suiteobj = get_suite(osuite)
 365         # Get a list of all suites that use the override file of 'osuite'
 366         ocodename = suiteobj.codename
 367         suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()]
 368
 369         if len(suiteids) < 1:
 370             utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
 371
 372         for component in cnf.SubTree("Component").List():
 373             # It is crucial for the dsc override creation based on binary
 374             # overrides that 'dsc' goes first
 375             otypes = cnf.ValueList("OverrideType")
 376             otypes.remove("dsc")
 377             otypes = ["dsc"] + otypes
 378             for otype in otypes:
 379                 print "Processing %s [%s - %s]" \
 380                     % (osuite, component, otype)
 381                 sys.stdout.flush()
 382                 process(osuite, suiteids, originosuite, component, otype, session)
 383
 384     Logger.close()
 385
 386 ################################################################################
 387
 388 if __name__ == '__main__':
 389     main()