#!/usr/bin/env python # Check for obsolete binary packages # Copyright (C) 2000, 2001, 2002, 2003 James Troup # $Id: rene,v 1.19 2003-02-10 18:24:25 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ################################################################################ # ``If you're claiming that's a "problem" that needs to be "fixed", # you might as well write some letters to God about how unfair entropy # is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au ## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add support for non-US ?, add overrides, avoid ANAIS for duplicated packages ################################################################################ import commands, pg, os, string, sys, tempfile, time; import utils, db_access; import apt_pkg; ################################################################################ Cnf = None; projectB = None; suite_id = None; ################################################################################ def usage(exit_code=0): print """Usage: rene Check for obsolete or duplicated packages. -h, --help show this help and exit.""" sys.exit(exit_code) ################################################################################ def add_nbs(nbs_d, source, version, package): if not nbs_d.has_key(source): nbs_d[source] = {}; if not nbs_d[source].has_key(version): nbs_d[source][version] = {}; nbs_d[source][version][package] = ""; ################################################################################ def main (): global Cnf, projectB, suite_id; Cnf = utils.get_conf(); Arguments = [('h',"help","Rene::Options::Help")]; for i in [ "help" ]: if not Cnf.has_key("Rene::Options::%s" % (i)): Cnf["Rene::Options::%s" % (i)] = ""; apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); Options = Cnf.SubTree("Rene::Options") if Options["Help"]: usage(); projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init(Cnf, projectB); bin_pkgs = {}; src_pkgs = {}; source_binaries = {}; bins_in_suite = {}; nbs = {}; source_versions = {}; anais_output = ""; duplicate_bins = {}; suite = "unstable"; suite_id = db_access.get_suite_id(suite); bin_not_built = {}; # Initalize a large hash table of all binary packages before = time.time(); sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)); q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = 5 AND ba.bin = b.id"); ql = q.getresult(); sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); for i in ql: bins_in_suite[i[0]] = ""; components = Cnf.ValueList("Suite::%s::Components" % (suite)); for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component); # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... temp_filename = tempfile.mktemp(); fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700); os.close(fd); (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)); if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)); sys.exit(result); sources = utils.open_file(temp_filename); Sources = apt_pkg.ParseTagFile(sources); while Sources.Step(): source = Sources.Section.Find('Package'); source_version = Sources.Section.Find('Version'); architecture = Sources.Section.Find('Architecture'); binaries = Sources.Section.Find('Binary'); binaries_list = map(string.strip, binaries.split(',')); # Check for binaries not built on any architecture. for binary in binaries_list: if not bins_in_suite.has_key(binary): if not bin_not_built.has_key(source): bin_not_built[source] = {}; bin_not_built[source][binary] = ""; # Check for packages built on architectures they shouldn't be. if architecture != "any" and architecture != "all": architectures = {}; for arch in architecture.split(): architectures[arch.strip()] = ""; for binary in binaries_list: q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary)); ql = q.getresult(); versions = []; for i in ql: arch = i[0]; version = i[1]; if architectures.has_key(arch): versions.append(version); versions.sort(apt_pkg.VersionCompare); if versions: latest_version = versions.pop() else: latest_version = None; # Check for 'invalid' architectures versions_d = {} for i in ql: arch = i[0]; version = i[1]; if not architectures.has_key(arch): if not versions_d.has_key(version): versions_d[version] = []; versions_d[version].append(arch) if versions_d != {}: anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture); versions = versions_d.keys(); versions.sort(apt_pkg.VersionCompare); for version in versions: arches = versions_d[version]; arches.sort(); anais_output += " o %s: %s\n" % (version, ", ".join(arches)); # Check for duplicated packages and build indices for checking "no source" later source_index = component + '/' + source; if src_pkgs.has_key(source): print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]); src_pkgs[source] = source_index; for binary in binaries_list: if bin_pkgs.has_key(binary): key = "%s~%s" % (source, bin_pkgs[binary]); if not duplicate_bins.has_key(key): duplicate_bins[key] = []; duplicate_bins[key].append(binary); bin_pkgs[binary] = source; source_binaries[source] = binaries; source_versions[source] = source_version; sources.close(); os.unlink(temp_filename); for component in components: architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))); for architecture in architectures: filename = "%s/dists/%s/%s/binary-%s/Packages" % (Cnf["Dir::Root"], suite, component, architecture); packages = utils.open_file(filename); Packages = apt_pkg.ParseTagFile(packages); while Packages.Step(): package = Packages.Section.Find('Package'); source = Packages.Section.Find('Source', ""); version = Packages.Section.Find('Version'); if source == "": source = package; if source.find("(") != -1: m = utils.re_extract_src_version.match(source); source = m.group(1); version = m.group(2); if not bin_pkgs.has_key(package): if not nbs.has_key(source): nbs[source] = {}; if not nbs[source].has_key(package): nbs[source][package] = {}; nbs[source][package][version] = ""; packages.close(); dubious_nbs = {}; real_nbs = {}; for source in nbs.keys(): for package in nbs[source].keys(): versions = nbs[source][package].keys(); versions.sort(apt_pkg.VersionCompare); latest_version = versions.pop(); source_version = source_versions.get(source,"0"); if apt_pkg.VersionCompare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package); else: add_nbs(real_nbs, source, latest_version, package); # Check for packages in experimental obsoleted by versions in unstable suite_id = db_access.get_suite_id("unstable"); q = projectB.query(""" SELECT s.source, s.version AS experimental, s2.version AS unstable FROM src_associations sa, source s, source s2, src_associations sa2 WHERE sa.suite = 1 AND sa2.suite = %d AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source AND versioncmp(s.version, s2.version) < 0""" % (suite_id)); ql = q.getresult(); if ql: nviu_to_remove = []; print "Newer version in unstable"; print "-------------------------"; print ; for i in ql: (source, experimental_version, unstable_version) = i; print " o %s (%s, %s)" % (source, experimental_version, unstable_version); nviu_to_remove.append(source); print print "Suggested command:" print " melanie -m \"[rene] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove)); print print "Not Built from Source"; print "---------------------"; print ; nbs_to_remove = []; nbs_keys = real_nbs.keys(); nbs_keys.sort(); for source in nbs_keys: binaries = source_binaries.get(source, "(source does not exist)") print " * %s_%s builds: %s" % (source, source_versions.get(source, "??"), source_binaries.get(source, "(source does not exist)")); print " but no longer builds:" versions = real_nbs[source].keys(); versions.sort(apt_pkg.VersionCompare); for version in versions: packages = real_nbs[source][version].keys(); packages.sort(); for pkg in packages: # *cough* FIXME if pkg.find("pcmcia") == -1: nbs_to_remove.append(pkg); print " o %s: %s" % (version, ", ".join(packages)); print ; if nbs_to_remove: print "Suggested command:" print " melanie -m \"[rene] NBS\" -b %s" % (" ".join(nbs_to_remove)); print print "="*75 print print "Unbuilt binary packages"; print "-----------------------"; print keys = bin_not_built.keys(); keys.sort(); for source in keys: binaries = bin_not_built[source].keys(); binaries.sort(); print " o %s: %s" % (source, ", ".join(binaries)); print ; print "Built from multiple source packages"; print "-----------------------------------"; print ; keys = duplicate_bins.keys(); keys.sort(); for key in keys: (source_a, source_b) = key.split("~"); print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key])); print ; print "Architecture Not Allowed In Source"; print "----------------------------------"; print anais_output; print ; print "Dubious NBS"; print "-----------"; print ; dubious_nbs_keys = dubious_nbs.keys(); dubious_nbs_keys.sort(); for source in dubious_nbs_keys: binaries = source_binaries.get(source, "(source does not exist)") print " * %s_%s builds: %s" % (source, source_versions.get(source, "??"), source_binaries.get(source, "(source does not exist)")); print " won't admit to building:" versions = dubious_nbs[source].keys(); versions.sort(apt_pkg.VersionCompare); for version in versions: packages = dubious_nbs[source][version].keys(); packages.sort(); print " o %s: %s" % (version, ", ".join(packages)); print ; ################################################################################ if __name__ == '__main__': main()