3 # Cruft checker and hole filler for overrides
4 # Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
5 # Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 ######################################################################
24 # NB: dak check-overrides is not a good idea with New Incoming as it #
25 # doesn't take into account accepted. You can minimize the impact #
26 # of this by running it immediately after dak process-accepted but #
27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
28 # process-accepted'. A better long term fix is the evil plan for #
29 # accepted to be in the DB. #
30 ######################################################################
32 # dak check-overrides should now work fine being done during
33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
34 # process-accepted' and 'dak make-suite-file-list'). At that point,
35 # queue/accepted should be empty and installed, so... dak
36 # check-overrides does now take into account suites sharing overrides
39 # * Only update out-of-sync overrides when corresponding versions are equal to
41 # * consistency checks like:
42 # - section=debian-installer only for udeb and # dsc
43 # - priority=source iff dsc
44 # - (suite, package, 'dsc') is unique,
45 # - just as (suite, package, (u)deb) (yes, across components!)
46 # - sections match their component (each component has an own set of sections,
47 # could probably be reduced...)
49 ################################################################################
52 import dak.lib.utils, dak.lib.database, dak.lib.logging
55 ################################################################################
64 ################################################################################
66 def usage (exit_code=0):
67 print """Usage: dak check-overrides
68 Check for cruft in overrides.
70 -n, --no-action don't do anything
71 -h, --help show this help and exit"""
75 ################################################################################
77 def gen_blacklist(dir):
78 for entry in os.listdir(dir):
79 entry = entry.split('_')[0]
82 def process(osuite, affected_suites, originosuite, component, type):
83 global Logger, Options, projectB, sections, priorities
85 osuite_id = dak.lib.database.get_suite_id(osuite)
87 dak.lib.utils.fubar("Suite '%s' not recognised." % (osuite))
88 originosuite_id = None
90 originosuite_id = dak.lib.database.get_suite_id(originosuite)
91 if originosuite_id == -1:
92 dak.lib.utils.fubar("Suite '%s' not recognised." % (originosuite))
94 component_id = dak.lib.database.get_component_id(component)
95 if component_id == -1:
96 dak.lib.utils.fubar("Component '%s' not recognised." % (component))
98 type_id = dak.lib.database.get_override_type_id(type)
100 dak.lib.utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
101 dsc_type_id = dak.lib.database.get_override_type_id("dsc")
102 deb_type_id = dak.lib.database.get_override_type_id("deb")
104 source_priority_id = dak.lib.database.get_priority_id("source")
106 if type == "deb" or type == "udeb":
108 q = projectB.query("""
109 SELECT b.package FROM binaries b, bin_associations ba, files f,
110 location l, component c
111 WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
112 AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
113 """ % (type, ",".join(map(str,affected_suites)), component_id))
114 for i in q.getresult():
118 q = projectB.query("""
119 SELECT s.source FROM source s, src_associations sa, files f, location l,
121 WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
122 AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
123 """ % (",".join(map(str,affected_suites)), component_id))
124 for i in q.getresult():
125 src_packages[i[0]] = 0
128 # Drop unused overrides
130 q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
131 projectB.query("BEGIN WORK")
133 for i in q.getresult():
135 if src_packages.has_key(package):
136 src_packages[package] = 1
138 if blacklist.has_key(package):
139 dak.lib.utils.warn("%s in incoming, not touching" % package)
141 Logger.log(["removing unused override", osuite, component,
142 type, package, priorities[i[1]], sections[i[2]], i[3]])
143 if not Options["No-Action"]:
144 projectB.query("""DELETE FROM override WHERE package =
145 '%s' AND suite = %s AND component = %s AND type =
146 %s""" % (package, osuite_id, component_id, type_id))
147 # create source overrides based on binary overrides, as source
148 # overrides not always get created
149 q = projectB.query(""" SELECT package, priority, section,
150 maintainer FROM override WHERE suite = %s AND component = %s
151 """ % (osuite_id, component_id))
152 for i in q.getresult():
154 if not src_packages.has_key(package) or src_packages[package]:
156 src_packages[package] = 1
158 Logger.log(["add missing override", osuite, component,
159 type, package, "source", sections[i[2]], i[3]])
160 if not Options["No-Action"]:
161 projectB.query("""INSERT INTO override (package, suite,
162 component, priority, section, type, maintainer) VALUES
163 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
164 osuite_id, component_id, source_priority_id, i[2],
166 # Check whether originosuite has an override for us we can
169 q = projectB.query("""SELECT origin.package, origin.priority,
170 origin.section, origin.maintainer, target.priority,
171 target.section, target.maintainer FROM override origin LEFT
172 JOIN override target ON (origin.package = target.package AND
173 target.suite=%s AND origin.component = target.component AND origin.type =
174 target.type) WHERE origin.suite = %s AND origin.component = %s
175 AND origin.type = %s""" %
176 (osuite_id, originosuite_id, component_id, type_id))
177 for i in q.getresult():
179 if not src_packages.has_key(package) or src_packages[package]:
180 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
181 Logger.log(["syncing override", osuite, component,
182 type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
183 if not Options["No-Action"]:
184 projectB.query("""UPDATE override SET section=%s,
185 maintainer='%s' WHERE package='%s' AND
186 suite=%s AND component=%s AND type=%s""" %
187 (i[2], i[3], package, osuite_id, component_id,
191 src_packages[package] = 1
192 Logger.log(["copying missing override", osuite, component,
193 type, package, "source", sections[i[2]], i[3]])
194 if not Options["No-Action"]:
195 projectB.query("""INSERT INTO override (package, suite,
196 component, priority, section, type, maintainer) VALUES
197 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
198 osuite_id, component_id, source_priority_id, i[2],
201 for package, hasoverride in src_packages.items():
203 dak.lib.utils.warn("%s has no override!" % package)
205 else: # binary override
206 for i in q.getresult():
208 if packages.has_key(package):
209 packages[package] = 1
211 if blacklist.has_key(package):
212 dak.lib.utils.warn("%s in incoming, not touching" % package)
214 Logger.log(["removing unused override", osuite, component,
215 type, package, priorities[i[1]], sections[i[2]], i[3]])
216 if not Options["No-Action"]:
217 projectB.query("""DELETE FROM override WHERE package =
218 '%s' AND suite = %s AND component = %s AND type =
219 %s""" % (package, osuite_id, component_id, type_id))
221 # Check whether originosuite has an override for us we can
224 q = projectB.query("""SELECT origin.package, origin.priority,
225 origin.section, origin.maintainer, target.priority,
226 target.section, target.maintainer FROM override origin LEFT
227 JOIN override target ON (origin.package = target.package AND
228 target.suite=%s AND origin.component = target.component AND
229 origin.type = target.type) WHERE origin.suite = %s AND
230 origin.component = %s AND origin.type = %s""" % (osuite_id,
231 originosuite_id, component_id, type_id))
232 for i in q.getresult():
234 if not packages.has_key(package) or packages[package]:
235 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
236 Logger.log(["syncing override", osuite, component,
237 type, package, priorities[i[4]], sections[i[5]],
238 i[6], priorities[i[1]], sections[i[2]], i[3]])
239 if not Options["No-Action"]:
240 projectB.query("""UPDATE override SET priority=%s, section=%s,
241 maintainer='%s' WHERE package='%s' AND
242 suite=%s AND component=%s AND type=%s""" %
243 (i[1], i[2], i[3], package, osuite_id,
244 component_id, type_id))
247 packages[package] = 1
248 Logger.log(["copying missing override", osuite, component,
249 type, package, priorities[i[1]], sections[i[2]], i[3]])
250 if not Options["No-Action"]:
251 projectB.query("""INSERT INTO override (package, suite,
252 component, priority, section, type, maintainer) VALUES
253 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
255 for package, hasoverride in packages.items():
257 dak.lib.utils.warn("%s has no override!" % package)
259 projectB.query("COMMIT WORK")
263 ################################################################################
266 global Logger, Options, projectB, sections, priorities
268 Cnf = dak.lib.utils.get_conf()
270 Arguments = [('h',"help","Check-Overrides::Options::Help"),
271 ('n',"no-action", "Check-Overrides::Options::No-Action")]
272 for i in [ "help", "no-action" ]:
273 if not Cnf.has_key("Check-Overrides::Options::%s" % (i)):
274 Cnf["Check-Overrides::Options::%s" % (i)] = ""
275 apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
276 Options = Cnf.SubTree("Check-Overrides::Options")
281 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
282 dak.lib.database.init(Cnf, projectB)
284 # init sections, priorities:
285 q = projectB.query("SELECT id, section FROM section")
286 for i in q.getresult():
287 sections[i[0]] = i[1]
288 q = projectB.query("SELECT id, priority FROM priority")
289 for i in q.getresult():
290 priorities[i[0]] = i[1]
292 if not Options["No-Action"]:
293 Logger = dak.lib.logging.Logger(Cnf, "check-overrides")
295 Logger = dak.lib.logging.Logger(Cnf, "check-overrides", 1)
297 gen_blacklist(Cnf["Dir::Queue::Accepted"])
299 for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
300 if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
303 osuite = osuite.lower()
308 originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
309 originosuite = originosuite.lower()
310 originremark = " taking missing from %s" % originosuite
314 print "Processing %s%s..." % (osuite, originremark)
315 # Get a list of all suites that use the override file of 'osuite'
316 ocodename = Cnf["Suite::%s::codename" % osuite]
318 for suite in Cnf.SubTree("Suite").List():
319 if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]:
322 q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
323 % ", ".join(map(repr, suites)).lower())
326 for i in q.getresult():
327 suiteids.append(i[0])
329 if len(suiteids) != len(suites) or len(suiteids) < 1:
330 dak.lib.utils.fubar("Couldn't find id's of all suites: %s" % suites)
332 for component in Cnf.SubTree("Component").List():
333 if component == "mixed":
335 # It is crucial for the dsc override creation based on binary
336 # overrides that 'dsc' goes first
337 otypes = Cnf.ValueList("OverrideType")
339 otypes = ["dsc"] + otypes
341 print "Processing %s [%s - %s] using %s..." \
342 % (osuite, component, otype, suites)
344 process(osuite, suiteids, originosuite, component, otype)
348 ################################################################################
350 if __name__ == '__main__':