]> git.decadent.org.uk Git - dak.git/blob - dak/check_overrides.py
Merge remote branch 'drkranz/master' into merge
[dak.git] / dak / check_overrides.py
1 #!/usr/bin/env python
2
3 """ Cruft checker and hole filler for overrides """
4 # Copyright (C) 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
5 # Copyright (C) 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
6
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20
21 ################################################################################
22
23 ######################################################################
24 # NB: dak check-overrides is not a good idea with New Incoming as it #
25 # doesn't take into account accepted.  You can minimize the impact   #
26 # of this by running it immediately after dak process-accepted but   #
27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
28 # process-accepted'.  A better long term fix is the evil plan for    #
29 # accepted to be in the DB.                                          #
30 ######################################################################
31
32 # dak check-overrides should now work fine being done during
33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
34 # process-accepted' and 'dak make-suite-file-list'). At that point,
35 # queue/accepted should be empty and installed, so... dak
36 # check-overrides does now take into account suites sharing overrides
37
38 # TODO:
39 # * Only update out-of-sync overrides when corresponding versions are equal to
40 #   some degree
41 # * consistency checks like:
42 #   - section=debian-installer only for udeb and # dsc
43 #   - priority=source iff dsc
44 #   - (suite, package, 'dsc') is unique,
45 #   - just as (suite, package, (u)deb) (yes, across components!)
46 #   - sections match their component (each component has an own set of sections,
47 #     could probably be reduced...)
48
49 ################################################################################
50
51 import sys, os
52 import apt_pkg
53
54 from daklib.config import Config
55 from daklib.dbconn import *
56 from daklib import daklog
57 from daklib import utils
58
59 ################################################################################
60
61 Options = None
62 Logger = None
63 sections = {}
64 priorities = {}
65 blacklist = {}
66
67 ################################################################################
68
69 def usage (exit_code=0):
70     print """Usage: dak check-overrides
71 Check for cruft in overrides.
72
73   -n, --no-action            don't do anything
74   -h, --help                 show this help and exit"""
75
76     sys.exit(exit_code)
77
78 ################################################################################
79
80 def process(osuite, affected_suites, originosuite, component, otype, session):
81     global Logger, Options, sections, priorities
82
83     o = get_suite(osuite, session)
84     if o is None:
85         utils.fubar("Suite '%s' not recognised." % (osuite))
86     osuite_id = o.suite_id
87
88     originosuite_id = None
89     if originosuite:
90         oo = get_suite(originosuite, session)
91         if oo is None:
92             utils.fubar("Suite '%s' not recognised." % (originosuite))
93         originosuite_id = oo.suite_id
94
95     c = get_component(component, session)
96     if c is None:
97         utils.fubar("Component '%s' not recognised." % (component))
98     component_id = c.component_id
99
100     ot = get_override_type(otype, session)
101     if ot is None:
102         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
103     type_id = ot.overridetype_id
104     dsc_type_id = get_override_type("dsc", session).overridetype_id
105
106     source_priority_id = get_priority("source", session).priority_id
107
108     if otype == "deb" or otype == "udeb":
109         packages = {}
110         # TODO: Fix to use placeholders (check how to with arrays)
111         q = session.execute("""
112 SELECT b.package FROM binaries b, bin_associations ba, files f,
113                               location l, component c
114  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
115    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
116 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
117         for i in q.fetchall():
118             packages[i[0]] = 0
119
120     src_packages = {}
121     q = session.execute("""
122 SELECT s.source FROM source s, src_associations sa, files f, location l,
123                      component c
124  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
125    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
126 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
127     for i in q.fetchall():
128         src_packages[i[0]] = 0
129
130     # -----------
131     # Drop unused overrides
132
133     q = session.execute("""SELECT package, priority, section, maintainer
134                              FROM override WHERE suite = :suite_id
135                               AND component = :component_id AND type = :type_id""",
136                         {'suite_id': osuite_id, 'component_id': component_id,
137                          'type_id': type_id})
138     # We're already within a transaction
139     if otype == "dsc":
140         for i in q.fetchall():
141             package = i[0]
142             if src_packages.has_key(package):
143                 src_packages[package] = 1
144             else:
145                 if blacklist.has_key(package):
146                     utils.warn("%s in incoming, not touching" % package)
147                     continue
148                 Logger.log(["removing unused override", osuite, component,
149                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
150                 if not Options["No-Action"]:
151                     session.execute("""DELETE FROM override WHERE package = :package
152                                           AND suite = :suite_id AND component = :component_id
153                                           AND type = :type_id""",
154                                     {'package': package, 'suite_id': osuite_id,
155                                      'component_id': component_id, 'type_id': type_id})
156         # create source overrides based on binary overrides, as source
157         # overrides not always get created
158         q = session.execute("""SELECT package, priority, section, maintainer
159                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
160                             {'suite_id': osuite_id, 'component_id': component_id})
161         for i in q.fetchall():
162             package = i[0]
163             if not src_packages.has_key(package) or src_packages[package]:
164                 continue
165             src_packages[package] = 1
166
167             Logger.log(["add missing override", osuite, component,
168                 otype, package, "source", sections[i[2]], i[3]])
169             if not Options["No-Action"]:
170                 session.execute("""INSERT INTO override (package, suite, component,
171                                                         priority, section, type, maintainer)
172                                          VALUES (:package, :suite_id, :component_id,
173                                                  :priority_id, :section_id, :type_id, :maintainer)""",
174                                {'package': package, 'suite_id': osuite_id,
175                                 'component_id': component_id, 'priority_id': source_priority_id,
176                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
177         # Check whether originosuite has an override for us we can
178         # copy
179         if originosuite:
180             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
181                                          origin.maintainer, target.priority, target.section,
182                                          target.maintainer
183                                     FROM override origin
184                                LEFT JOIN override target ON (origin.package = target.package
185                                                              AND target.suite = :suite_id
186                                                              AND origin.component = target.component
187                                                              AND origin.type = target.type)
188                                    WHERE origin.suite = :originsuite_id
189                                      AND origin.component = :component_id
190                                      AND origin.type = :type_id""",
191                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
192                                  'component_id': component_id, 'type_id': type_id})
193             for i in q.fetchall():
194                 package = i[0]
195                 if not src_packages.has_key(package) or src_packages[package]:
196                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
197                         Logger.log(["syncing override", osuite, component,
198                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
199                         if not Options["No-Action"]:
200                             session.execute("""UPDATE override
201                                                  SET section = :section,
202                                                      maintainer = :maintainer
203                                                WHERE package = :package AND suite = :suite_id
204                                                  AND component = :component_id AND type = :type_id""",
205                                             {'section': i[2], 'maintainer': i[3],
206                                              'package': package, 'suite_id': osuite_id,
207                                              'component_id': component_id, 'type_id': dsc_type_id})
208                     continue
209
210                 # we can copy
211                 src_packages[package] = 1
212                 Logger.log(["copying missing override", osuite, component,
213                     otype, package, "source", sections[i[2]], i[3]])
214                 if not Options["No-Action"]:
215                     session.execute("""INSERT INTO override (package, suite, component,
216                                                              priority, section, type, maintainer)
217                                             VALUES (:package, :suite_id, :component_id,
218                                                     :priority_id, :section_id, :type_id,
219                                                     :maintainer)""",
220                                     {'package': package, 'suite_id': osuite_id,
221                                      'component_id': component_id, 'priority_id': source_priority_id,
222                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
223
224         for package, hasoverride in src_packages.items():
225             if not hasoverride:
226                 utils.warn("%s has no override!" % package)
227
228     else: # binary override
229         for i in q.fetchall():
230             package = i[0]
231             if packages.has_key(package):
232                 packages[package] = 1
233             else:
234                 if blacklist.has_key(package):
235                     utils.warn("%s in incoming, not touching" % package)
236                     continue
237                 Logger.log(["removing unused override", osuite, component,
238                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
239                 if not Options["No-Action"]:
240                     session.execute("""DELETE FROM override
241                                         WHERE package = :package AND suite = :suite_id
242                                           AND component = :component_id AND type = :type_id""",
243                                     {'package': package, 'suite_id': osuite_id,
244                                      'component_id': component_id, 'type_id': type_id})
245
246         # Check whether originosuite has an override for us we can
247         # copy
248         if originosuite:
249             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
250                                           origin.maintainer, target.priority, target.section,
251                                           target.maintainer
252                                      FROM override origin LEFT JOIN override target
253                                                           ON (origin.package = target.package
254                                                               AND target.suite = :suite_id
255                                                               AND origin.component = target.component
256                                                               AND origin.type = target.type)
257                                     WHERE origin.suite = :originsuite_id
258                                       AND origin.component = :component_id
259                                       AND origin.type = :type_id""",
260                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
261                                   'component_id': component_id, 'type_id': type_id})
262             for i in q.fetchall():
263                 package = i[0]
264                 if not packages.has_key(package) or packages[package]:
265                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
266                         Logger.log(["syncing override", osuite, component,
267                             otype, package, priorities[i[4]], sections[i[5]],
268                             i[6], priorities[i[1]], sections[i[2]], i[3]])
269                         if not Options["No-Action"]:
270                             session.execute("""UPDATE override
271                                                   SET priority = :priority_id,
272                                                       section = :section_id,
273                                                       maintainer = :maintainer
274                                                 WHERE package = :package
275                                                   AND suite = :suite_id
276                                                   AND component = :component_id
277                                                   AND type = :type_id""",
278                                             {'priority_id': i[1], 'section_id': i[2],
279                                              'maintainer': i[3], 'package': package,
280                                              'suite_id': osuite_id, 'component_id': component_id,
281                                              'type_id': type_id})
282                     continue
283                 # we can copy
284                 packages[package] = 1
285                 Logger.log(["copying missing override", osuite, component,
286                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
287                 if not Options["No-Action"]:
288                     session.execute("""INSERT INTO override (package, suite, component,
289                                                              priority, section, type, maintainer)
290                                             VALUES (:package, :suite_id, :component_id,
291                                                     :priority_id, :section_id, :type_id, :maintainer)""",
292                                     {'package': package, 'suite_id': osuite_id,
293                                      'component_id': component_id, 'priority_id': i[1],
294                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
295
296         for package, hasoverride in packages.items():
297             if not hasoverride:
298                 utils.warn("%s has no override!" % package)
299
300     session.commit()
301     sys.stdout.flush()
302
303
304 ################################################################################
305
306 def main ():
307     global Logger, Options, sections, priorities
308
309     cnf = Config()
310
311     Arguments = [('h',"help","Check-Overrides::Options::Help"),
312                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
313     for i in [ "help", "no-action" ]:
314         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
315             cnf["Check-Overrides::Options::%s" % (i)] = ""
316     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
317     Options = cnf.SubTree("Check-Overrides::Options")
318
319     if Options["Help"]:
320         usage()
321
322     session = DBConn().session()
323
324     # init sections, priorities:
325
326     # We need forward and reverse
327     sections = get_sections(session)
328     for name, entry in sections.items():
329         sections[entry] = name
330
331     priorities = get_priorities(session)
332     for name, entry in priorities.items():
333         priorities[entry] = name
334
335     if not Options["No-Action"]:
336         Logger = daklog.Logger(cnf, "check-overrides")
337     else:
338         Logger = daklog.Logger(cnf, "check-overrides", 1)
339
340     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
341         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
342             continue
343
344         osuite = osuite.lower()
345
346         originosuite = None
347         originremark = ""
348         try:
349             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
350             originosuite = originosuite.lower()
351             originremark = " taking missing from %s" % originosuite
352         except KeyError:
353             pass
354
355         print "Processing %s%s..." % (osuite, originremark)
356         # Get a list of all suites that use the override file of 'osuite'
357         ocodename = cnf["Suite::%s::codename" % osuite].lower()
358         suites = []
359         suiteids = []
360         for suite in cnf.SubTree("Suite").List():
361             if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
362                 suites.append(suite)
363                 s = get_suite(suite.lower(), session)
364                 if s is not None:
365                     suiteids.append(s.suite_id)
366
367         if len(suiteids) != len(suites) or len(suiteids) < 1:
368             utils.fubar("Couldn't find id's of all suites: %s" % suites)
369
370         for component in cnf.SubTree("Component").List():
371             # It is crucial for the dsc override creation based on binary
372             # overrides that 'dsc' goes first
373             otypes = cnf.ValueList("OverrideType")
374             otypes.remove("dsc")
375             otypes = ["dsc"] + otypes
376             for otype in otypes:
377                 print "Processing %s [%s - %s] using %s..." \
378                     % (osuite, component, otype, suites)
379                 sys.stdout.flush()
380                 process(osuite, suiteids, originosuite, component, otype, session)
381
382     Logger.close()
383
384 ################################################################################
385
386 if __name__ == '__main__':
387     main()