]> git.decadent.org.uk Git - dak.git/blob - dak/check_overrides.py
dont have GOODSIG in the output
[dak.git] / dak / check_overrides.py
1 #!/usr/bin/env python
2
3 """ Cruft checker and hole filler for overrides
4
5 @contact: Debian FTPMaster <ftpmaster@debian.org>
6 @copyright: 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
7 @opyright: 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
8 @copyright: 2011  Joerg Jaspert <joerg@debian.org>
9 @license: GNU General Public License version 2 or later
10
11 """
12
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
17
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 # GNU General Public License for more details.
22
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
26
27 ################################################################################
28
29 ######################################################################
30 # NB: dak check-overrides is not a good idea with New Incoming as it #
31 # doesn't take into account accepted.  You can minimize the impact   #
32 # of this by running it immediately after dak process-accepted but   #
33 # that's still racy because 'dak process-new' doesn't lock with 'dak #
34 # process-accepted'.  A better long term fix is the evil plan for    #
35 # accepted to be in the DB.                                          #
36 ######################################################################
37
38 # dak check-overrides should now work fine being done during
39 # cron.daily, for example just before 'dak make-overrides' (after 'dak
40 # process-accepted' and 'dak make-suite-file-list'). At that point,
41 # queue/accepted should be empty and installed, so... dak
42 # check-overrides does now take into account suites sharing overrides
43
44 # TODO:
45 # * Only update out-of-sync overrides when corresponding versions are equal to
46 #   some degree
47 # * consistency checks like:
48 #   - section=debian-installer only for udeb and # dsc
49 #   - priority=source iff dsc
50 #   - (suite, package, 'dsc') is unique,
51 #   - just as (suite, package, (u)deb) (yes, across components!)
52 #   - sections match their component (each component has an own set of sections,
53 #     could probably be reduced...)
54
55 ################################################################################
56
57 import sys, os
58 import apt_pkg
59
60 from daklib.config import Config
61 from daklib.dbconn import *
62 from daklib import daklog
63 from daklib import utils
64
65 ################################################################################
66
67 Options = None                 #: Commandline arguments parsed into this
68 Logger = None                  #: Our logging object
69 sections = {}
70 priorities = {}
71 blacklist = {}
72
73 ################################################################################
74
75 def usage (exit_code=0):
76     print """Usage: dak check-overrides
77 Check for cruft in overrides.
78
79   -n, --no-action            don't do anything
80   -h, --help                 show this help and exit"""
81
82     sys.exit(exit_code)
83
84 ################################################################################
85
86 def process(osuite, affected_suites, originosuite, component, otype, session):
87     global Logger, Options, sections, priorities
88
89     o = get_suite(osuite, session)
90     if o is None:
91         utils.fubar("Suite '%s' not recognised." % (osuite))
92     osuite_id = o.suite_id
93
94     originosuite_id = None
95     if originosuite:
96         oo = get_suite(originosuite, session)
97         if oo is None:
98             utils.fubar("Suite '%s' not recognised." % (originosuite))
99         originosuite_id = oo.suite_id
100
101     c = get_component(component, session)
102     if c is None:
103         utils.fubar("Component '%s' not recognised." % (component))
104     component_id = c.component_id
105
106     ot = get_override_type(otype, session)
107     if ot is None:
108         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
109     type_id = ot.overridetype_id
110     dsc_type_id = get_override_type("dsc", session).overridetype_id
111
112     source_priority_id = get_priority("source", session).priority_id
113
114     if otype == "deb" or otype == "udeb":
115         packages = {}
116         # TODO: Fix to use placeholders (check how to with arrays)
117         q = session.execute("""
118 SELECT b.package FROM binaries b, bin_associations ba, files f,
119                               location l, component c
120  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
121    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
122 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
123         for i in q.fetchall():
124             packages[i[0]] = 0
125
126     src_packages = {}
127     q = session.execute("""
128 SELECT s.source FROM source s, src_associations sa, files f, location l,
129                      component c
130  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
131    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
132 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
133     for i in q.fetchall():
134         src_packages[i[0]] = 0
135
136     # -----------
137     # Drop unused overrides
138
139     q = session.execute("""SELECT package, priority, section, maintainer
140                              FROM override WHERE suite = :suite_id
141                               AND component = :component_id AND type = :type_id""",
142                         {'suite_id': osuite_id, 'component_id': component_id,
143                          'type_id': type_id})
144     # We're already within a transaction
145     if otype == "dsc":
146         for i in q.fetchall():
147             package = i[0]
148             if src_packages.has_key(package):
149                 src_packages[package] = 1
150             else:
151                 if blacklist.has_key(package):
152                     utils.warn("%s in incoming, not touching" % package)
153                     continue
154                 Logger.log(["removing unused override", osuite, component,
155                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
156                 if not Options["No-Action"]:
157                     session.execute("""DELETE FROM override WHERE package = :package
158                                           AND suite = :suite_id AND component = :component_id
159                                           AND type = :type_id
160                                           AND created < now() - interval '14 days'""",
161                                     {'package': package, 'suite_id': osuite_id,
162                                      'component_id': component_id, 'type_id': type_id})
163         # create source overrides based on binary overrides, as source
164         # overrides not always get created
165         q = session.execute("""SELECT package, priority, section, maintainer
166                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
167                             {'suite_id': osuite_id, 'component_id': component_id})
168         for i in q.fetchall():
169             package = i[0]
170             if not src_packages.has_key(package) or src_packages[package]:
171                 continue
172             src_packages[package] = 1
173
174             Logger.log(["add missing override", osuite, component,
175                 otype, package, "source", sections[i[2]], i[3]])
176             if not Options["No-Action"]:
177                 session.execute("""INSERT INTO override (package, suite, component,
178                                                         priority, section, type, maintainer)
179                                          VALUES (:package, :suite_id, :component_id,
180                                                  :priority_id, :section_id, :type_id, :maintainer)""",
181                                {'package': package, 'suite_id': osuite_id,
182                                 'component_id': component_id, 'priority_id': source_priority_id,
183                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
184         # Check whether originosuite has an override for us we can
185         # copy
186         if originosuite:
187             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
188                                          origin.maintainer, target.priority, target.section,
189                                          target.maintainer
190                                     FROM override origin
191                                LEFT JOIN override target ON (origin.package = target.package
192                                                              AND target.suite = :suite_id
193                                                              AND origin.component = target.component
194                                                              AND origin.type = target.type)
195                                    WHERE origin.suite = :originsuite_id
196                                      AND origin.component = :component_id
197                                      AND origin.type = :type_id""",
198                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
199                                  'component_id': component_id, 'type_id': type_id})
200             for i in q.fetchall():
201                 package = i[0]
202                 if not src_packages.has_key(package) or src_packages[package]:
203                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
204                         Logger.log(["syncing override", osuite, component,
205                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
206                         if not Options["No-Action"]:
207                             session.execute("""UPDATE override
208                                                  SET section = :section,
209                                                      maintainer = :maintainer
210                                                WHERE package = :package AND suite = :suite_id
211                                                  AND component = :component_id AND type = :type_id""",
212                                             {'section': i[2], 'maintainer': i[3],
213                                              'package': package, 'suite_id': osuite_id,
214                                              'component_id': component_id, 'type_id': dsc_type_id})
215                     continue
216
217                 # we can copy
218                 src_packages[package] = 1
219                 Logger.log(["copying missing override", osuite, component,
220                     otype, package, "source", sections[i[2]], i[3]])
221                 if not Options["No-Action"]:
222                     session.execute("""INSERT INTO override (package, suite, component,
223                                                              priority, section, type, maintainer)
224                                             VALUES (:package, :suite_id, :component_id,
225                                                     :priority_id, :section_id, :type_id,
226                                                     :maintainer)""",
227                                     {'package': package, 'suite_id': osuite_id,
228                                      'component_id': component_id, 'priority_id': source_priority_id,
229                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
230
231         for package, hasoverride in src_packages.items():
232             if not hasoverride:
233                 utils.warn("%s has no override!" % package)
234
235     else: # binary override
236         for i in q.fetchall():
237             package = i[0]
238             if packages.has_key(package):
239                 packages[package] = 1
240             else:
241                 if blacklist.has_key(package):
242                     utils.warn("%s in incoming, not touching" % package)
243                     continue
244                 Logger.log(["removing unused override", osuite, component,
245                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
246                 if not Options["No-Action"]:
247                     session.execute("""DELETE FROM override
248                                         WHERE package = :package AND suite = :suite_id
249                                           AND component = :component_id AND type = :type_id
250                                           AND created < now() - interval '14 days'""",
251                                     {'package': package, 'suite_id': osuite_id,
252                                      'component_id': component_id, 'type_id': type_id})
253
254         # Check whether originosuite has an override for us we can
255         # copy
256         if originosuite:
257             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
258                                           origin.maintainer, target.priority, target.section,
259                                           target.maintainer
260                                      FROM override origin LEFT JOIN override target
261                                                           ON (origin.package = target.package
262                                                               AND target.suite = :suite_id
263                                                               AND origin.component = target.component
264                                                               AND origin.type = target.type)
265                                     WHERE origin.suite = :originsuite_id
266                                       AND origin.component = :component_id
267                                       AND origin.type = :type_id""",
268                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
269                                   'component_id': component_id, 'type_id': type_id})
270             for i in q.fetchall():
271                 package = i[0]
272                 if not packages.has_key(package) or packages[package]:
273                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
274                         Logger.log(["syncing override", osuite, component,
275                             otype, package, priorities[i[4]], sections[i[5]],
276                             i[6], priorities[i[1]], sections[i[2]], i[3]])
277                         if not Options["No-Action"]:
278                             session.execute("""UPDATE override
279                                                   SET priority = :priority_id,
280                                                       section = :section_id,
281                                                       maintainer = :maintainer
282                                                 WHERE package = :package
283                                                   AND suite = :suite_id
284                                                   AND component = :component_id
285                                                   AND type = :type_id""",
286                                             {'priority_id': i[1], 'section_id': i[2],
287                                              'maintainer': i[3], 'package': package,
288                                              'suite_id': osuite_id, 'component_id': component_id,
289                                              'type_id': type_id})
290                     continue
291                 # we can copy
292                 packages[package] = 1
293                 Logger.log(["copying missing override", osuite, component,
294                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
295                 if not Options["No-Action"]:
296                     session.execute("""INSERT INTO override (package, suite, component,
297                                                              priority, section, type, maintainer)
298                                             VALUES (:package, :suite_id, :component_id,
299                                                     :priority_id, :section_id, :type_id, :maintainer)""",
300                                     {'package': package, 'suite_id': osuite_id,
301                                      'component_id': component_id, 'priority_id': i[1],
302                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
303
304         for package, hasoverride in packages.items():
305             if not hasoverride:
306                 utils.warn("%s has no override!" % package)
307
308     session.commit()
309     sys.stdout.flush()
310
311
312 ################################################################################
313
314 def main ():
315     global Logger, Options, sections, priorities
316
317     cnf = Config()
318
319     Arguments = [('h',"help","Check-Overrides::Options::Help"),
320                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
321     for i in [ "help", "no-action" ]:
322         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
323             cnf["Check-Overrides::Options::%s" % (i)] = ""
324     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
325     Options = cnf.SubTree("Check-Overrides::Options")
326
327     if Options["Help"]:
328         usage()
329
330     session = DBConn().session()
331
332     # init sections, priorities:
333
334     # We need forward and reverse
335     sections = get_sections(session)
336     for name, entry in sections.items():
337         sections[entry] = name
338
339     priorities = get_priorities(session)
340     for name, entry in priorities.items():
341         priorities[entry] = name
342
343     if not Options["No-Action"]:
344         Logger = daklog.Logger(cnf, "check-overrides")
345     else:
346         Logger = daklog.Logger(cnf, "check-overrides", 1)
347
348     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
349         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
350             continue
351
352         osuite = osuite.lower()
353
354         originosuite = None
355         originremark = ""
356         try:
357             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
358             originosuite = originosuite.lower()
359             originremark = " taking missing from %s" % originosuite
360         except KeyError:
361             pass
362
363         print "Processing %s%s..." % (osuite, originremark)
364         suiteobj = get_suite(osuite)
365         # Get a list of all suites that use the override file of 'osuite'
366         ocodename = suiteobj.codename
367         suiteids = [x.suite_id for x in session.query(Suite).filter(Suite.overridecodename == ocodename).all()]
368
369         if len(suiteids) < 1:
370             utils.fubar("Couldn't find id's of all suites: %s" % suiteids)
371
372         for component in cnf.SubTree("Component").List():
373             # It is crucial for the dsc override creation based on binary
374             # overrides that 'dsc' goes first
375             otypes = cnf.ValueList("OverrideType")
376             otypes.remove("dsc")
377             otypes = ["dsc"] + otypes
378             for otype in otypes:
379                 print "Processing %s [%s - %s]" \
380                     % (osuite, component, otype)
381                 sys.stdout.flush()
382                 process(osuite, suiteids, originosuite, component, otype, session)
383
384     Logger.close()
385
386 ################################################################################
387
388 if __name__ == '__main__':
389     main()