3 # Check for obsolete binary packages
4 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 ################################################################################
22 # ``If you're claiming that's a "problem" that needs to be "fixed",
23 # you might as well write some letters to God about how unfair entropy
24 # is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
26 ## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add overrides, avoid ANAIS for duplicated packages
28 ################################################################################
30 import commands, pg, os, sys, time
32 import daklib.database
35 ################################################################################
40 no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy
45 ################################################################################
47 def usage(exit_code=0):
48 print """Usage: dak cruft-report
49 Check for obsolete or duplicated packages.
51 -h, --help show this help and exit.
52 -m, --mode=MODE chose the MODE to run in (full or daily).
53 -s, --suite=SUITE check suite SUITE."""
56 ################################################################################
58 def add_nbs(nbs_d, source, version, package):
59 # Ensure the package is still in the suite (someone may have already removed it)
60 if no_longer_in_suite.has_key(package):
63 q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package))
65 no_longer_in_suite[package] = ""
68 nbs_d.setdefault(source, {})
69 nbs_d[source].setdefault(version, {})
70 nbs_d[source][version][package] = ""
72 ################################################################################
74 # Check for packages built on architectures they shouldn't be.
75 def do_anais(architecture, binaries_list, source):
76 if architecture == "any" or architecture == "all":
81 for arch in architecture.split():
82 architectures[arch.strip()] = ""
83 for binary in binaries_list:
84 q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary))
90 if architectures.has_key(arch):
91 versions.append(version)
92 versions.sort(apt_pkg.VersionCompare)
94 latest_version = versions.pop()
97 # Check for 'invalid' architectures
102 if not architectures.has_key(arch):
103 versions_d.setdefault(version, [])
104 versions_d[version].append(arch)
107 anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture)
108 versions = versions_d.keys()
109 versions.sort(apt_pkg.VersionCompare)
110 for version in versions:
111 arches = versions_d[version]
113 anais_output += " o %s: %s\n" % (version, ", ".join(arches))
116 ################################################################################
119 experimental_id = daklib.database.get_suite_id("experimental")
120 if experimental_id == -1:
122 # Check for packages in experimental obsoleted by versions in unstable
123 q = projectB.query("""
124 SELECT s.source, s.version AS experimental, s2.version AS unstable
125 FROM src_associations sa, source s, source s2, src_associations sa2
126 WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id
127 AND sa2.source = s2.id AND s.source = s2.source
128 AND versioncmp(s.version, s2.version) < 0""" % (experimental_id,
129 daklib.database.get_suite_id("unstable")))
133 print "Newer version in unstable"
134 print "-------------------------"
137 (source, experimental_version, unstable_version) = i
138 print " o %s (%s, %s)" % (source, experimental_version, unstable_version)
139 nviu_to_remove.append(source)
141 print "Suggested command:"
142 print " dak rm -m \"[auto-cruft] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove))
145 ################################################################################
147 def do_nbs(real_nbs):
148 output = "Not Built from Source\n"
149 output += "---------------------\n\n"
152 nbs_keys = real_nbs.keys()
154 for source in nbs_keys:
155 output += " * %s_%s builds: %s\n" % (source,
156 source_versions.get(source, "??"),
157 source_binaries.get(source, "(source does not exist)"))
158 output += " but no longer builds:\n"
159 versions = real_nbs[source].keys()
160 versions.sort(apt_pkg.VersionCompare)
161 for version in versions:
162 packages = real_nbs[source][version].keys()
165 nbs_to_remove.append(pkg)
166 output += " o %s: %s\n" % (version, ", ".join(packages))
173 print "Suggested command:"
174 print " dak rm -m \"[auto-cruft] NBS\" -b %s" % (" ".join(nbs_to_remove))
177 ################################################################################
179 def do_dubious_nbs(dubious_nbs):
184 dubious_nbs_keys = dubious_nbs.keys()
185 dubious_nbs_keys.sort()
186 for source in dubious_nbs_keys:
187 print " * %s_%s builds: %s" % (source,
188 source_versions.get(source, "??"),
189 source_binaries.get(source, "(source does not exist)"))
190 print " won't admit to building:"
191 versions = dubious_nbs[source].keys()
192 versions.sort(apt_pkg.VersionCompare)
193 for version in versions:
194 packages = dubious_nbs[source][version].keys()
196 print " o %s: %s" % (version, ", ".join(packages))
200 ################################################################################
202 def do_obsolete_source(duplicate_bins, bin2source):
204 for key in duplicate_bins.keys():
205 (source_a, source_b) = key.split('_')
206 for source in [ source_a, source_b ]:
207 if not obsolete.has_key(source):
208 if not source_binaries.has_key(source):
209 # Source has already been removed
212 obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ]
213 for binary in duplicate_bins[key]:
214 if bin2source.has_key(binary) and bin2source[binary]["source"] == source:
216 if binary in obsolete[source]:
217 obsolete[source].remove(binary)
220 output = "Obsolete source package\n"
221 output += "-----------------------\n\n"
222 obsolete_keys = obsolete.keys()
224 for source in obsolete_keys:
225 if not obsolete[source]:
226 to_remove.append(source)
227 output += " * %s (%s)\n" % (source, source_versions[source])
228 for binary in [ i.strip() for i in source_binaries[source].split(',') ]:
229 if bin2source.has_key(binary):
230 output += " o %s (%s) is built by %s.\n" \
231 % (binary, bin2source[binary]["version"],
232 bin2source[binary]["source"])
234 output += " o %s is not built.\n" % binary
240 print "Suggested command:"
241 print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
244 ################################################################################
247 global Cnf, projectB, suite_id, source_binaries, source_versions
249 Cnf = daklib.utils.get_conf()
251 Arguments = [('h',"help","Cruft-Report::Options::Help"),
252 ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
253 ('s',"suite","Cruft-Report::Options::Suite","HasArg")]
255 if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
256 Cnf["Cruft-Report::Options::%s" % (i)] = ""
257 Cnf["Cruft-Report::Options::Suite"] = Cnf["Dinstall::DefaultSuite"]
259 if not Cnf.has_key("Cruft-Report::Options::Mode"):
260 Cnf["Cruft-Report::Options::Mode"] = "daily"
262 apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
264 Options = Cnf.SubTree("Cruft-Report::Options")
268 # Set up checks based on mode
269 if Options["Mode"] == "daily":
270 checks = [ "nbs", "nviu", "obsolete source" ]
271 elif Options["Mode"] == "full":
272 checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]
274 daklib.utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
277 projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
278 daklib.database.init(Cnf, projectB)
290 suite = Options["Suite"]
291 suite_id = daklib.database.get_suite_id(suite)
296 # Initalize a large hash table of all binary packages
298 sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
299 q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
301 sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
303 bins_in_suite[i[0]] = ""
305 # Checks based on the Sources files
306 components = Cnf.ValueList("Suite::%s::Components" % (suite))
307 for component in components:
308 filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component)
309 # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
310 temp_filename = daklib.utils.temp_filename()
311 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
313 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
315 sources = daklib.utils.open_file(temp_filename)
316 Sources = apt_pkg.ParseTagFile(sources)
317 while Sources.Step():
318 source = Sources.Section.Find('Package')
319 source_version = Sources.Section.Find('Version')
320 architecture = Sources.Section.Find('Architecture')
321 binaries = Sources.Section.Find('Binary')
322 binaries_list = [ i.strip() for i in binaries.split(',') ]
325 # Check for binaries not built on any architecture.
326 for binary in binaries_list:
327 if not bins_in_suite.has_key(binary):
328 bin_not_built.setdefault(source, {})
329 bin_not_built[source][binary] = ""
331 if "anais" in checks:
332 anais_output += do_anais(architecture, binaries_list, source)
334 # Check for duplicated packages and build indices for checking "no source" later
335 source_index = component + '/' + source
336 if src_pkgs.has_key(source):
337 print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source])
338 src_pkgs[source] = source_index
339 for binary in binaries_list:
340 if bin_pkgs.has_key(binary):
341 key_list = [ source, bin_pkgs[binary] ]
343 key = '_'.join(key_list)
344 duplicate_bins.setdefault(key, [])
345 duplicate_bins[key].append(binary)
346 bin_pkgs[binary] = source
347 source_binaries[source] = binaries
348 source_versions[source] = source_version
351 os.unlink(temp_filename)
353 # Checks based on the Packages files
354 for component in components + ['main/debian-installer']:
355 architectures = filter(daklib.utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite)))
356 for architecture in architectures:
357 filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture)
358 # apt_pkg.ParseTagFile needs a real file handle
359 temp_filename = daklib.utils.temp_filename()
360 (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
362 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
364 packages = daklib.utils.open_file(temp_filename)
365 Packages = apt_pkg.ParseTagFile(packages)
366 while Packages.Step():
367 package = Packages.Section.Find('Package')
368 source = Packages.Section.Find('Source', "")
369 version = Packages.Section.Find('Version')
372 if bin2source.has_key(package) and \
373 apt_pkg.VersionCompare(version, bin2source[package]["version"]) > 0:
374 bin2source[package]["version"] = version
375 bin2source[package]["source"] = source
377 bin2source[package] = {}
378 bin2source[package]["version"] = version
379 bin2source[package]["source"] = source
380 if source.find("(") != -1:
381 m = daklib.utils.re_extract_src_version.match(source)
384 if not bin_pkgs.has_key(package):
385 nbs.setdefault(source,{})
386 nbs[source].setdefault(package, {})
387 nbs[source][package][version] = ""
389 previous_source = bin_pkgs[package]
390 if previous_source != source:
391 key_list = [ source, previous_source ]
393 key = '_'.join(key_list)
394 duplicate_bins.setdefault(key, [])
395 if package not in duplicate_bins[key]:
396 duplicate_bins[key].append(package)
398 os.unlink(temp_filename)
400 if "obsolete source" in checks:
401 do_obsolete_source(duplicate_bins, bin2source)
403 # Distinguish dubious (version numbers match) and 'real' NBS (they don't)
406 for source in nbs.keys():
407 for package in nbs[source].keys():
408 versions = nbs[source][package].keys()
409 versions.sort(apt_pkg.VersionCompare)
410 latest_version = versions.pop()
411 source_version = source_versions.get(source,"0")
412 if apt_pkg.VersionCompare(latest_version, source_version) == 0:
413 add_nbs(dubious_nbs, source, latest_version, package)
415 add_nbs(real_nbs, source, latest_version, package)
425 if Options["Mode"] == "full":
430 print "Unbuilt binary packages"
431 print "-----------------------"
433 keys = bin_not_built.keys()
436 binaries = bin_not_built[source].keys()
438 print " o %s: %s" % (source, ", ".join(binaries))
442 print "Built from multiple source packages"
443 print "-----------------------------------"
445 keys = duplicate_bins.keys()
448 (source_a, source_b) = key.split("_")
449 print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key]))
452 if "anais" in checks:
453 print "Architecture Not Allowed In Source"
454 print "----------------------------------"
458 if "dubious nbs" in checks:
459 do_dubious_nbs(dubious_nbs)
462 ################################################################################
464 if __name__ == '__main__':