3 # Cruft checker and hole filler for overrides
4 # Copyright (C) 2000, 2001, 2002, 2004 James Troup <james@nocrew.org>
5 # Copyright (C) 2005 Jeroen van Wolffelaar <jeroen@wolffelaar.nl>
6 # $Id: cindy,v 1.14 2005-11-15 09:50:32 ajt Exp $
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 ################################################################################
24 ######################################################################
25 # NB: cindy is not a good idea with New Incoming as she doesn't take #
26 # into account accepted. You can minimize the impact of this by #
27 # running her immediately after kelly but that's still racy because #
28 # lisa doesn't lock with kelly. A better long term fix is the evil #
29 # plan for accepted to be in the DB. #
30 ######################################################################
32 # cindy should now work fine being done during cron.daily, for example just
33 # before denise (after kelly and jenna). At that point, queue/accepted should
34 # be empty and installed, so... Cindy does now take into account suites
38 # * Only update out-of-sync overrides when corresponding versions are equal to
40 # * consistency checks like:
41 # - section=debian-installer only for udeb and # dsc
42 # - priority=source iff dsc
43 # - (suite, package, 'dsc') is unique,
44 # - just as (suite, package, (u)deb) (yes, across components!)
45 # - sections match their component (each component has an own set of sections,
46 # could probably be reduced...)
48 ################################################################################
51 import utils, db_access, logging
54 ################################################################################
63 ################################################################################
65 def usage (exit_code=0):
67 Check for cruft in overrides.
69 -n, --no-action don't do anything
70 -h, --help show this help and exit"""
74 ################################################################################
76 def gen_blacklist(dir):
77 for entry in os.listdir(dir):
78 entry = entry.split('_')[0]
81 def process(osuite, affected_suites, originosuite, component, type):
82 global Logger, Options, projectB, sections, priorities
84 osuite_id = db_access.get_suite_id(osuite)
86 utils.fubar("Suite '%s' not recognised." % (osuite))
87 originosuite_id = None
89 originosuite_id = db_access.get_suite_id(originosuite)
90 if originosuite_id == -1:
91 utils.fubar("Suite '%s' not recognised." % (originosuite))
93 component_id = db_access.get_component_id(component)
94 if component_id == -1:
95 utils.fubar("Component '%s' not recognised." % (component))
97 type_id = db_access.get_override_type_id(type)
99 utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type))
100 dsc_type_id = db_access.get_override_type_id("dsc")
101 deb_type_id = db_access.get_override_type_id("deb")
103 source_priority_id = db_access.get_priority_id("source")
105 if type == "deb" or type == "udeb":
107 q = projectB.query("""
108 SELECT b.package FROM binaries b, bin_associations ba, files f,
109 location l, component c
110 WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location
111 AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s
112 """ % (type, ",".join(map(str,affected_suites)), component_id))
113 for i in q.getresult():
117 q = projectB.query("""
118 SELECT s.source FROM source s, src_associations sa, files f, location l,
120 WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location
121 AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s
122 """ % (",".join(map(str,affected_suites)), component_id))
123 for i in q.getresult():
124 src_packages[i[0]] = 0
127 # Drop unused overrides
129 q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id))
130 projectB.query("BEGIN WORK")
132 for i in q.getresult():
134 if src_packages.has_key(package):
135 src_packages[package] = 1
137 if blacklist.has_key(package):
138 utils.warn("%s in incoming, not touching" % package)
140 Logger.log(["removing unused override", osuite, component,
141 type, package, priorities[i[1]], sections[i[2]], i[3]])
142 if not Options["No-Action"]:
143 projectB.query("""DELETE FROM override WHERE package =
144 '%s' AND suite = %s AND component = %s AND type =
145 %s""" % (package, osuite_id, component_id, type_id))
146 # create source overrides based on binary overrides, as source
147 # overrides not always get created
148 q = projectB.query(""" SELECT package, priority, section,
149 maintainer FROM override WHERE suite = %s AND component = %s
150 """ % (osuite_id, component_id))
151 for i in q.getresult():
153 if not src_packages.has_key(package) or src_packages[package]:
155 src_packages[package] = 1
157 Logger.log(["add missing override", osuite, component,
158 type, package, "source", sections[i[2]], i[3]])
159 if not Options["No-Action"]:
160 projectB.query("""INSERT INTO override (package, suite,
161 component, priority, section, type, maintainer) VALUES
162 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
163 osuite_id, component_id, source_priority_id, i[2],
165 # Check whether originosuite has an override for us we can
168 q = projectB.query("""SELECT origin.package, origin.priority,
169 origin.section, origin.maintainer, target.priority,
170 target.section, target.maintainer FROM override origin LEFT
171 JOIN override target ON (origin.package = target.package AND
172 target.suite=%s AND origin.component = target.component AND origin.type =
173 target.type) WHERE origin.suite = %s AND origin.component = %s
174 AND origin.type = %s""" %
175 (osuite_id, originosuite_id, component_id, type_id))
176 for i in q.getresult():
178 if not src_packages.has_key(package) or src_packages[package]:
179 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
180 Logger.log(["syncing override", osuite, component,
181 type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]])
182 if not Options["No-Action"]:
183 projectB.query("""UPDATE override SET section=%s,
184 maintainer='%s' WHERE package='%s' AND
185 suite=%s AND component=%s AND type=%s""" %
186 (i[2], i[3], package, osuite_id, component_id,
190 src_packages[package] = 1
191 Logger.log(["copying missing override", osuite, component,
192 type, package, "source", sections[i[2]], i[3]])
193 if not Options["No-Action"]:
194 projectB.query("""INSERT INTO override (package, suite,
195 component, priority, section, type, maintainer) VALUES
196 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package,
197 osuite_id, component_id, source_priority_id, i[2],
200 for package, hasoverride in src_packages.items():
202 utils.warn("%s has no override!" % package)
204 else: # binary override
205 for i in q.getresult():
207 if packages.has_key(package):
208 packages[package] = 1
210 if blacklist.has_key(package):
211 utils.warn("%s in incoming, not touching" % package)
213 Logger.log(["removing unused override", osuite, component,
214 type, package, priorities[i[1]], sections[i[2]], i[3]])
215 if not Options["No-Action"]:
216 projectB.query("""DELETE FROM override WHERE package =
217 '%s' AND suite = %s AND component = %s AND type =
218 %s""" % (package, osuite_id, component_id, type_id))
220 # Check whether originosuite has an override for us we can
223 q = projectB.query("""SELECT origin.package, origin.priority,
224 origin.section, origin.maintainer, target.priority,
225 target.section, target.maintainer FROM override origin LEFT
226 JOIN override target ON (origin.package = target.package AND
227 target.suite=%s AND origin.component = target.component AND
228 origin.type = target.type) WHERE origin.suite = %s AND
229 origin.component = %s AND origin.type = %s""" % (osuite_id,
230 originosuite_id, component_id, type_id))
231 for i in q.getresult():
233 if not packages.has_key(package) or packages[package]:
234 if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]):
235 Logger.log(["syncing override", osuite, component,
236 type, package, priorities[i[4]], sections[i[5]],
237 i[6], priorities[i[1]], sections[i[2]], i[3]])
238 if not Options["No-Action"]:
239 projectB.query("""UPDATE override SET priority=%s, section=%s,
240 maintainer='%s' WHERE package='%s' AND
241 suite=%s AND component=%s AND type=%s""" %
242 (i[1], i[2], i[3], package, osuite_id,
243 component_id, type_id))
246 packages[package] = 1
247 Logger.log(["copying missing override", osuite, component,
248 type, package, priorities[i[1]], sections[i[2]], i[3]])
249 if not Options["No-Action"]:
250 projectB.query("""INSERT INTO override (package, suite,
251 component, priority, section, type, maintainer) VALUES
252 ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3]))
254 for package, hasoverride in packages.items():
256 utils.warn("%s has no override!" % package)
258 projectB.query("COMMIT WORK")
262 ################################################################################
265 global Logger, Options, projectB, sections, priorities
267 Cnf = utils.get_conf()
269 Arguments = [('h',"help","Cindy::Options::Help"),
270 ('n',"no-action", "Cindy::Options::No-Action")]
271 for i in [ "help", "no-action" ]:
272 if not Cnf.has_key("Cindy::Options::%s" % (i)):
273 Cnf["Cindy::Options::%s" % (i)] = ""
274 apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
275 Options = Cnf.SubTree("Cindy::Options")
280 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
281 db_access.init(Cnf, projectB)
283 # init sections, priorities:
284 q = projectB.query("SELECT id, section FROM section")
285 for i in q.getresult():
286 sections[i[0]] = i[1]
287 q = projectB.query("SELECT id, priority FROM priority")
288 for i in q.getresult():
289 priorities[i[0]] = i[1]
291 if not Options["No-Action"]:
292 Logger = logging.Logger(Cnf, "cindy")
294 Logger = logging.Logger(Cnf, "cindy", 1)
296 gen_blacklist(Cnf["Dir::Queue::Accepted"])
298 for osuite in Cnf.SubTree("Cindy::OverrideSuites").List():
299 if "1" != Cnf["Cindy::OverrideSuites::%s::Process" % osuite]:
302 osuite = osuite.lower()
307 originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite]
308 originosuite = originosuite.lower()
309 originremark = " taking missing from %s" % originosuite
313 print "Processing %s%s..." % (osuite, originremark)
314 # Get a list of all suites that use the override file of 'osuite'
315 ocodename = Cnf["Suite::%s::codename" % osuite]
317 for suite in Cnf.SubTree("Suite").List():
318 if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]:
321 q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \
322 % ", ".join(map(repr, suites)).lower())
325 for i in q.getresult():
326 suiteids.append(i[0])
328 if len(suiteids) != len(suites) or len(suiteids) < 1:
329 utils.fubar("Couldn't find id's of all suites: %s" % suites)
331 for component in Cnf.SubTree("Component").List():
332 if component == "mixed":
334 # It is crucial for the dsc override creation based on binary
335 # overrides that 'dsc' goes first
336 otypes = Cnf.ValueList("OverrideType")
338 otypes = ["dsc"] + otypes
340 print "Processing %s [%s - %s] using %s..." \
341 % (osuite, component, otype, suites)
343 process(osuite, suiteids, originosuite, component, otype)
347 ################################################################################
349 if __name__ == '__main__':