]> git.decadent.org.uk Git - dak.git/blob - dak/check_overrides.py
Merge branch 'merge'
[dak.git] / dak / check_overrides.py
1 #!/usr/bin/env python
2
3 """ Cruft checker and hole filler for overrides """
4 # Copyright (C) 2000, 2001, 2002, 2004, 2006  James Troup <james@nocrew.org>
5 # Copyright (C) 2005  Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
6
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20
21 ################################################################################
22
23 ######################################################################
24 # NB: dak check-overrides is not a good idea with New Incoming as it #
25 # doesn't take into account accepted.  You can minimize the impact   #
26 # of this by running it immediately after dak process-accepted but   #
27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
28 # process-accepted'.  A better long term fix is the evil plan for    #
29 # accepted to be in the DB.                                          #
30 ######################################################################
31
32 # dak check-overrides should now work fine being done during
33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
34 # process-accepted' and 'dak make-suite-file-list'). At that point,
35 # queue/accepted should be empty and installed, so... dak
36 # check-overrides does now take into account suites sharing overrides
37
38 # TODO:
39 # * Only update out-of-sync overrides when corresponding versions are equal to
40 #   some degree
41 # * consistency checks like:
42 #   - section=debian-installer only for udeb and # dsc
43 #   - priority=source iff dsc
44 #   - (suite, package, 'dsc') is unique,
45 #   - just as (suite, package, (u)deb) (yes, across components!)
46 #   - sections match their component (each component has an own set of sections,
47 #     could probably be reduced...)
48
49 ################################################################################
50
51 import sys, os
52 import apt_pkg
53
54 from daklib.config import Config
55 from daklib.dbconn import *
56 from daklib import daklog
57 from daklib import utils
58
59 ################################################################################
60
61 Options = None
62 Logger = None
63 sections = {}
64 priorities = {}
65 blacklist = {}
66
67 ################################################################################
68
69 def usage (exit_code=0):
70     print """Usage: dak check-overrides
71 Check for cruft in overrides.
72
73   -n, --no-action            don't do anything
74   -h, --help                 show this help and exit"""
75
76     sys.exit(exit_code)
77
78 ################################################################################
79
80 def gen_blacklist(dir):
81     for entry in os.listdir(dir):
82         entry = entry.split('_')[0]
83         blacklist[entry] = 1
84
85 def process(osuite, affected_suites, originosuite, component, otype, session):
86     global Logger, Options, sections, priorities
87
88     o = get_suite(osuite, session)
89     if o is None:
90         utils.fubar("Suite '%s' not recognised." % (osuite))
91     osuite_id = o.suite_id
92
93     originosuite_id = None
94     if originosuite:
95         oo = get_suite(originosuite, session)
96         if oo is None:
97             utils.fubar("Suite '%s' not recognised." % (originosuite))
98         originosuite_id = oo.suite_id
99
100     c = get_component(component, session)
101     if c is None:
102         utils.fubar("Component '%s' not recognised." % (component))
103     component_id = c.component_id
104
105     ot = get_override_type(otype, session)
106     if ot is None:
107         utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (otype))
108     type_id = ot.overridetype_id
109     dsc_type_id = get_override_type("dsc", session).overridetype_id
110
111     source_priority_id = get_priority("source", session).priority_id
112
113     if otype == "deb" or otype == "udeb":
114         packages = {}
115         # TODO: Fix to use placeholders (check how to with arrays)
116         q = session.execute("""
117 SELECT b.package FROM binaries b, bin_associations ba, files f,
118                               location l, component c
119  WHERE b.type = :otype AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
120    AND c.id = l.component AND ba.suite IN (%s) AND c.id = :component_id
121 """ % (",".join([ str(i) for i in affected_suites ])), {'otype': otype, 'component_id': component_id})
122         for i in q.fetchall():
123             packages[i[0]] = 0
124
125     src_packages = {}
126     q = session.execute("""
127 SELECT s.source FROM source s, src_associations sa, files f, location l,
128                      component c
129  WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
130    AND c.id = l.component AND sa.suite IN (%s) AND c.id = :component_id
131 """ % (",".join([ str(i) for i in affected_suites])), {'component_id': component_id})
132     for i in q.fetchall():
133         src_packages[i[0]] = 0
134
135     # -----------
136     # Drop unused overrides
137
138     q = session.execute("""SELECT package, priority, section, maintainer
139                              FROM override WHERE suite = :suite_id
140                               AND component = :component_id AND type = :type_id""",
141                         {'suite_id': osuite_id, 'component_id': component_id,
142                          'type_id': type_id})
143     # We're already within a transaction
144     if otype == "dsc":
145         for i in q.fetchall():
146             package = i[0]
147             if src_packages.has_key(package):
148                 src_packages[package] = 1
149             else:
150                 if blacklist.has_key(package):
151                     utils.warn("%s in incoming, not touching" % package)
152                     continue
153                 Logger.log(["removing unused override", osuite, component,
154                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
155                 if not Options["No-Action"]:
156                     session.execute("""DELETE FROM override WHERE package = :package
157                                           AND suite = :suite_id AND component = :component_id
158                                           AND type = :type_id""",
159                                     {'package': package, 'suite_id': osuite_id,
160                                      'component_id': component_id, 'type_id': type_id})
161         # create source overrides based on binary overrides, as source
162         # overrides not always get created
163         q = session.execute("""SELECT package, priority, section, maintainer
164                                  FROM override WHERE suite = :suite_id AND component = :component_id""",
165                             {'suite_id': osuite_id, 'component_id': component_id})
166         for i in q.fetchall():
167             package = i[0]
168             if not src_packages.has_key(package) or src_packages[package]:
169                 continue
170             src_packages[package] = 1
171
172             Logger.log(["add missing override", osuite, component,
173                 otype, package, "source", sections[i[2]], i[3]])
174             if not Options["No-Action"]:
175                 session.execute("""INSERT INTO override (package, suite, component,
176                                                         priority, section, type, maintainer)
177                                          VALUES (:package, :suite_id, :component_id,
178                                                  :priority_id, :section_id, :type_id, :maintainer)""",
179                                {'package': package, 'suite_id': osuite_id,
180                                 'component_id': component_id, 'priority_id': source_priority_id,
181                                 'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
182         # Check whether originosuite has an override for us we can
183         # copy
184         if originosuite:
185             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
186                                          origin.maintainer, target.priority, target.section,
187                                          target.maintainer
188                                     FROM override origin
189                                LEFT JOIN override target ON (origin.package = target.package
190                                                              AND target.suite = :suite_id
191                                                              AND origin.component = target.component
192                                                              AND origin.type = target.type)
193                                    WHERE origin.suite = :originsuite_id
194                                      AND origin.component = :component_id
195                                      AND origin.type = :type_id""",
196                                 {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
197                                  'component_id': component_id, 'type_id': type_id})
198             for i in q.fetchall():
199                 package = i[0]
200                 if not src_packages.has_key(package) or src_packages[package]:
201                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
202                         Logger.log(["syncing override", osuite, component,
203                             otype, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
204                         if not Options["No-Action"]:
205                             session.execute("""UPDATE override
206                                                  SET section = :section,
207                                                      maintainer = :maintainer
208                                                WHERE package = :package AND suite = :suite_id
209                                                  AND component = :component_id AND type = :type_id""",
210                                             {'section': i[2], 'maintainer': i[3],
211                                              'package': package, 'suite_id': osuite_id,
212                                              'component_id': component_id, 'type_id': dsc_type_id})
213                     continue
214
215                 # we can copy
216                 src_packages[package] = 1
217                 Logger.log(["copying missing override", osuite, component,
218                     otype, package, "source", sections[i[2]], i[3]])
219                 if not Options["No-Action"]:
220                     session.execute("""INSERT INTO override (package, suite, component,
221                                                              priority, section, type, maintainer)
222                                             VALUES (:package, :suite_id, :component_id,
223                                                     :priority_id, :section_id, :type_id,
224                                                     :maintainer)""",
225                                     {'package': package, 'suite_id': osuite_id,
226                                      'component_id': component_id, 'priority_id': source_priority_id,
227                                      'section_id': i[2], 'type_id': dsc_type_id, 'maintainer': i[3]})
228
229         for package, hasoverride in src_packages.items():
230             if not hasoverride:
231                 utils.warn("%s has no override!" % package)
232
233     else: # binary override
234         for i in q.fetchall():
235             package = i[0]
236             if packages.has_key(package):
237                 packages[package] = 1
238             else:
239                 if blacklist.has_key(package):
240                     utils.warn("%s in incoming, not touching" % package)
241                     continue
242                 Logger.log(["removing unused override", osuite, component,
243                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
244                 if not Options["No-Action"]:
245                     session.execute("""DELETE FROM override
246                                         WHERE package = :package AND suite = :suite_id
247                                           AND component = :component_id AND type = :type_id""",
248                                     {'package': package, 'suite_id': osuite_id,
249                                      'component_id': component_id, 'type_id': type_id})
250
251         # Check whether originosuite has an override for us we can
252         # copy
253         if originosuite:
254             q = session.execute("""SELECT origin.package, origin.priority, origin.section,
255                                           origin.maintainer, target.priority, target.section,
256                                           target.maintainer
257                                      FROM override origin LEFT JOIN override target
258                                                           ON (origin.package = target.package
259                                                               AND target.suite = :suite_id
260                                                               AND origin.component = target.component
261                                                               AND origin.type = target.type)
262                                     WHERE origin.suite = :originsuite_id
263                                       AND origin.component = :component_id
264                                       AND origin.type = :type_id""",
265                                  {'suite_id': osuite_id, 'originsuite_id': originosuite_id,
266                                   'component_id': component_id, 'type_id': type_id})
267             for i in q.fetchall():
268                 package = i[0]
269                 if not packages.has_key(package) or packages[package]:
270                     if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
271                         Logger.log(["syncing override", osuite, component,
272                             otype, package, priorities[i[4]], sections[i[5]],
273                             i[6], priorities[i[1]], sections[i[2]], i[3]])
274                         if not Options["No-Action"]:
275                             session.execute("""UPDATE override
276                                                   SET priority = :priority_id,
277                                                       section = :section_id,
278                                                       maintainer = :maintainer
279                                                 WHERE package = :package
280                                                   AND suite = :suite_id
281                                                   AND component = :component_id
282                                                   AND type = :type_id""",
283                                             {'priority_id': i[1], 'section_id': i[2],
284                                              'maintainer': i[3], 'package': package,
285                                              'suite_id': osuite_id, 'component_id': component_id,
286                                              'type_id': type_id})
287                     continue
288                 # we can copy
289                 packages[package] = 1
290                 Logger.log(["copying missing override", osuite, component,
291                     otype, package, priorities[i[1]], sections[i[2]], i[3]])
292                 if not Options["No-Action"]:
293                     session.execute("""INSERT INTO override (package, suite, component,
294                                                              priority, section, type, maintainer)
295                                             VALUES (:package, :suite_id, :component_id,
296                                                     :priority_id, :section_id, :type_id, :maintainer)""",
297                                     {'package': package, 'suite_id': osuite_id,
298                                      'component_id': component_id, 'priority_id': i[1],
299                                      'section_id': i[2], 'type_id': type_id, 'maintainer': i[3]})
300
301         for package, hasoverride in packages.items():
302             if not hasoverride:
303                 utils.warn("%s has no override!" % package)
304
305     session.commit()
306     sys.stdout.flush()
307
308
309 ################################################################################
310
311 def main ():
312     global Logger, Options, sections, priorities
313
314     cnf = Config()
315
316     Arguments = [('h',"help","Check-Overrides::Options::Help"),
317                  ('n',"no-action", "Check-Overrides::Options::No-Action")]
318     for i in [ "help", "no-action" ]:
319         if not cnf.has_key("Check-Overrides::Options::%s" % (i)):
320             cnf["Check-Overrides::Options::%s" % (i)] = ""
321     apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
322     Options = cnf.SubTree("Check-Overrides::Options")
323
324     if Options["Help"]:
325         usage()
326
327     session = DBConn().session()
328
329     # init sections, priorities:
330
331     # We need forward and reverse
332     sections = get_sections(session)
333     for name, entry in sections.items():
334         sections[entry] = name
335
336     priorities = get_priorities(session)
337     for name, entry in priorities.items():
338         priorities[entry] = name
339
340     if not Options["No-Action"]:
341         Logger = daklog.Logger(cnf, "check-overrides")
342     else:
343         Logger = daklog.Logger(cnf, "check-overrides", 1)
344
345     gen_blacklist(cnf["Dir::Queue::Accepted"])
346
347     for osuite in cnf.SubTree("Check-Overrides::OverrideSuites").List():
348         if "1" != cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
349             continue
350
351         osuite = osuite.lower()
352
353         originosuite = None
354         originremark = ""
355         try:
356             originosuite = cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
357             originosuite = originosuite.lower()
358             originremark = " taking missing from %s" % originosuite
359         except KeyError:
360             pass
361
362         print "Processing %s%s..." % (osuite, originremark)
363         # Get a list of all suites that use the override file of 'osuite'
364         ocodename = cnf["Suite::%s::codename" % osuite].lower()
365         suites = []
366         suiteids = []
367         for suite in cnf.SubTree("Suite").List():
368             if ocodename == cnf["Suite::%s::OverrideCodeName" % suite].lower():
369                 suites.append(suite)
370                 s = get_suite(suite.lower(), session)
371                 if s is not None:
372                     suiteids.append(s.suite_id)
373
374         if len(suiteids) != len(suites) or len(suiteids) < 1:
375             utils.fubar("Couldn't find id's of all suites: %s" % suites)
376
377         for component in cnf.SubTree("Component").List():
378             # It is crucial for the dsc override creation based on binary
379             # overrides that 'dsc' goes first
380             otypes = cnf.ValueList("OverrideType")
381             otypes.remove("dsc")
382             otypes = ["dsc"] + otypes
383             for otype in otypes:
384                 print "Processing %s [%s - %s] using %s..." \
385                     % (osuite, component, otype, suites)
386                 sys.stdout.flush()
387                 process(osuite, suiteids, originosuite, component, otype, session)
388
389     Logger.close()
390
391 ################################################################################
392
393 if __name__ == '__main__':
394     main()