3 # Cruft checker and hole filler for overrides
4 # Copyright (C) 2000, 2001, 2002, 2004, 2006 James Troup <james@nocrew.org>
5 # Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 ################################################################################
23 ######################################################################
24 # NB: dak check-overrides is not a good idea with New Incoming as it #
25 # doesn't take into account accepted. You can minimize the impact #
26 # of this by running it immediately after dak process-accepted but #
27 # that's still racy because 'dak process-new' doesn't lock with 'dak #
28 # process-accepted'. A better long term fix is the evil plan for #
29 # accepted to be in the DB. #
30 ######################################################################
32 # dak check-overrides should now work fine being done during
33 # cron.daily, for example just before 'dak make-overrides' (after 'dak
34 # process-accepted' and 'dak make-suite-file-list'). At that point,
35 # queue/accepted should be empty and installed, so... dak
36 # check-overrides does now take into account suites sharing overrides
39 # * Only update out-of-sync overrides when corresponding versions are equal to
41 # * consistency checks like:
42 # - section=debian-installer only for udeb and # dsc
43 # - priority=source iff dsc
44 # - (suite, package, 'dsc') is unique,
45 # - just as (suite, package, (u)deb) (yes, across components!)
46 # - sections match their component (each component has an own set of sections,
47 # could probably be reduced...)
49 ################################################################################
53 import daklib.database
57 ################################################################################
66 ################################################################################
68 def usage (exit_code=0):
69 print """Usage: dak check-overrides
70 Check for cruft in overrides.
72 -n, --no-action don't do anything
73 -h, --help show this help and exit"""
77 ################################################################################
79 def gen_blacklist(dir):
80 for entry in os.listdir(dir):
81 entry = entry.split('_')[0]
84 def process(osuite, affected_suites, originosuite, component, type):
85 global Logger, Options, projectB, sections, priorities
87 osuite_id = daklib.database.get_suite_id(osuite)
89 daklib.utils.fubar("Suite '%s' not recognised." % (osuite))
90 originosuite_id = None
92 originosuite_id = daklib.database.get_suite_id(originosuite)
93 if originosuite_id == -1:
94 daklib.utils.fubar("Suite '%s' not recognised." % (originosuite))
96 component_id = daklib.database.get_component_id(component)
97 if component_id == -1:
98 daklib.utils.fubar("Component '%s' not recognised." % (component))
100 type_id = daklib.database.get_override_type_id(type)
102 daklib.utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
103 dsc_type_id = daklib.database.get_override_type_id("dsc")
104 deb_type_id = daklib.database.get_override_type_id("deb")
106 source_priority_id = daklib.database.get_priority_id("source")
108 if type == "deb" or type == "udeb":
110 q = projectB.query("""
111 SELECT b.package FROM binaries b, bin_associations ba, files f,
112 location l, component c
113 WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
114 AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
115 """ % (type, ",".join([ str(i) for i in affected_suites ]), component_id))
116 for i in q.getresult():
120 q = projectB.query("""
121 SELECT s.source FROM source s, src_associations sa, files f, location l,
123 WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
124 AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
125 """ % (",".join([ str(i) for i in affected_suites]), component_id))
126 for i in q.getresult():
127 src_packages[i[0]] = 0
130 # Drop unused overrides
132 q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
133 projectB.query("BEGIN WORK")
135 for i in q.getresult():
137 if src_packages.has_key(package):
138 src_packages[package] = 1
140 if blacklist.has_key(package):
141 daklib.utils.warn("%s in incoming, not touching" % package)
143 Logger.log(["removing unused override", osuite, component,
144 type, package, priorities[i[1]], sections[i[2]], i[3]])
145 if not Options["No-Action"]:
146 projectB.query("""DELETE FROM override WHERE package =
147 '%s' AND suite = %s AND component = %s AND type =
148 %s""" % (package, osuite_id, component_id, type_id))
149 # create source overrides based on binary overrides, as source
150 # overrides not always get created
151 q = projectB.query(""" SELECT package, priority, section,
152 maintainer FROM override WHERE suite = %s AND component = %s
153 """ % (osuite_id, component_id))
154 for i in q.getresult():
156 if not src_packages.has_key(package) or src_packages[package]:
158 src_packages[package] = 1
160 Logger.log(["add missing override", osuite, component,
161 type, package, "source", sections[i[2]], i[3]])
162 if not Options["No-Action"]:
163 projectB.query("""INSERT INTO override (package, suite,
164 component, priority, section, type, maintainer) VALUES
165 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
166 osuite_id, component_id, source_priority_id, i[2],
168 # Check whether originosuite has an override for us we can
171 q = projectB.query("""SELECT origin.package, origin.priority,
172 origin.section, origin.maintainer, target.priority,
173 target.section, target.maintainer FROM override origin LEFT
174 JOIN override target ON (origin.package = target.package AND
175 target.suite=%s AND origin.component = target.component AND origin.type =
176 target.type) WHERE origin.suite = %s AND origin.component = %s
177 AND origin.type = %s""" %
178 (osuite_id, originosuite_id, component_id, type_id))
179 for i in q.getresult():
181 if not src_packages.has_key(package) or src_packages[package]:
182 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
183 Logger.log(["syncing override", osuite, component,
184 type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
185 if not Options["No-Action"]:
186 projectB.query("""UPDATE override SET section=%s,
187 maintainer='%s' WHERE package='%s' AND
188 suite=%s AND component=%s AND type=%s""" %
189 (i[2], i[3], package, osuite_id, component_id,
193 src_packages[package] = 1
194 Logger.log(["copying missing override", osuite, component,
195 type, package, "source", sections[i[2]], i[3]])
196 if not Options["No-Action"]:
197 projectB.query("""INSERT INTO override (package, suite,
198 component, priority, section, type, maintainer) VALUES
199 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
200 osuite_id, component_id, source_priority_id, i[2],
203 for package, hasoverride in src_packages.items():
205 daklib.utils.warn("%s has no override!" % package)
207 else: # binary override
208 for i in q.getresult():
210 if packages.has_key(package):
211 packages[package] = 1
213 if blacklist.has_key(package):
214 daklib.utils.warn("%s in incoming, not touching" % package)
216 Logger.log(["removing unused override", osuite, component,
217 type, package, priorities[i[1]], sections[i[2]], i[3]])
218 if not Options["No-Action"]:
219 projectB.query("""DELETE FROM override WHERE package =
220 '%s' AND suite = %s AND component = %s AND type =
221 %s""" % (package, osuite_id, component_id, type_id))
223 # Check whether originosuite has an override for us we can
226 q = projectB.query("""SELECT origin.package, origin.priority,
227 origin.section, origin.maintainer, target.priority,
228 target.section, target.maintainer FROM override origin LEFT
229 JOIN override target ON (origin.package = target.package AND
230 target.suite=%s AND origin.component = target.component AND
231 origin.type = target.type) WHERE origin.suite = %s AND
232 origin.component = %s AND origin.type = %s""" % (osuite_id,
233 originosuite_id, component_id, type_id))
234 for i in q.getresult():
236 if not packages.has_key(package) or packages[package]:
237 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
238 Logger.log(["syncing override", osuite, component,
239 type, package, priorities[i[4]], sections[i[5]],
240 i[6], priorities[i[1]], sections[i[2]], i[3]])
241 if not Options["No-Action"]:
242 projectB.query("""UPDATE override SET priority=%s, section=%s,
243 maintainer='%s' WHERE package='%s' AND
244 suite=%s AND component=%s AND type=%s""" %
245 (i[1], i[2], i[3], package, osuite_id,
246 component_id, type_id))
249 packages[package] = 1
250 Logger.log(["copying missing override", osuite, component,
251 type, package, priorities[i[1]], sections[i[2]], i[3]])
252 if not Options["No-Action"]:
253 projectB.query("""INSERT INTO override (package, suite,
254 component, priority, section, type, maintainer) VALUES
255 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
257 for package, hasoverride in packages.items():
259 daklib.utils.warn("%s has no override!" % package)
261 projectB.query("COMMIT WORK")
265 ################################################################################
268 global Logger, Options, projectB, sections, priorities
270 Cnf = daklib.utils.get_conf()
272 Arguments = [('h',"help","Check-Overrides::Options::Help"),
273 ('n',"no-action", "Check-Overrides::Options::No-Action")]
274 for i in [ "help", "no-action" ]:
275 if not Cnf.has_key("Check-Overrides::Options::%s" % (i)):
276 Cnf["Check-Overrides::Options::%s" % (i)] = ""
277 apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
278 Options = Cnf.SubTree("Check-Overrides::Options")
283 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
284 daklib.database.init(Cnf, projectB)
286 # init sections, priorities:
287 q = projectB.query("SELECT id, section FROM section")
288 for i in q.getresult():
289 sections[i[0]] = i[1]
290 q = projectB.query("SELECT id, priority FROM priority")
291 for i in q.getresult():
292 priorities[i[0]] = i[1]
294 if not Options["No-Action"]:
295 Logger = daklib.logging.Logger(Cnf, "check-overrides")
297 Logger = daklib.logging.Logger(Cnf, "check-overrides", 1)
299 gen_blacklist(Cnf["Dir::Queue::Accepted"])
301 for osuite in Cnf.SubTree("Check-Overrides::OverrideSuites").List():
302 if "1" != Cnf["Check-Overrides::OverrideSuites::%s::Process" % osuite]:
305 osuite = osuite.lower()
310 originosuite = Cnf["Check-Overrides::OverrideSuites::%s::OriginSuite" % osuite]
311 originosuite = originosuite.lower()
312 originremark = " taking missing from %s" % originosuite
316 print "Processing %s%s..." % (osuite, originremark)
317 # Get a list of all suites that use the override file of 'osuite'
318 ocodename = Cnf["Suite::%s::codename" % osuite]
320 for suite in Cnf.SubTree("Suite").List():
321 if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]:
324 q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
325 % ", ".join([ repr(i) for i in suites ]).lower())
328 for i in q.getresult():
329 suiteids.append(i[0])
331 if len(suiteids) != len(suites) or len(suiteids) < 1:
332 daklib.utils.fubar("Couldn't find id's of all suites: %s" % suites)
334 for component in Cnf.SubTree("Component").List():
335 if component == "mixed":
337 # It is crucial for the dsc override creation based on binary
338 # overrides that 'dsc' goes first
339 otypes = Cnf.ValueList("OverrideType")
341 otypes = ["dsc"] + otypes
343 print "Processing %s [%s - %s] using %s..." \
344 % (osuite, component, otype, suites)
346 process(osuite, suiteids, originosuite, component, otype)
350 ################################################################################
352 if __name__ == '__main__':