From: James Troup <james@nocrew.org>
Date: Mon, 20 Jan 2003 19:13:21 +0000 (+0000)
Subject: Loads of changes to improve output
X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=3ca8a114bb074ff029e99cf28d308550f74f4c4f;p=dak.git

Loads of changes to improve output
---

diff --git a/rene b/rene
index e9a9c04b..d221ad03 100755
--- a/rene
+++ b/rene
@@ -2,7 +2,7 @@
 
 # Check for obsolete binary packages
 # Copyright (C) 2000, 2001, 2002  James Troup <james@nocrew.org>
-# $Id: rene,v 1.16 2003-01-02 18:10:02 troup Exp $
+# $Id: rene,v 1.17 2003-01-20 19:13:21 troup Exp $
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,9 +24,11 @@
 #   you might as well write some letters to God about how unfair entropy
 #   is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au
 
+## TODO:  fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add support for non-US ?, add overrides, avoid ANAIS for duplicated packages
+
 ################################################################################
 
-import commands, pg, os, sys, tempfile;
+import commands, pg, os, string, sys, tempfile, time;
 import utils, db_access;
 import apt_pkg;
 
@@ -34,6 +36,7 @@ import apt_pkg;
 
 Cnf = None;
 projectB = None;
+suite_id = None;
 
 ################################################################################
 
@@ -46,8 +49,17 @@ Check for obsolete or duplicated packages.
 
 ################################################################################
 
+def add_nbs(nbs_d, source, version, package):
+    if not nbs_d.has_key(source):
+        nbs_d[source] = {};
+    if not nbs_d[source].has_key(version):
+        nbs_d[source][version] = {};
+    nbs_d[source][version][package] = "";
+
+################################################################################
+
 def main ():
-    global Cnf, projectB;
+    global Cnf, projectB, suite_id;
 
     Cnf = utils.get_conf();
 
@@ -66,13 +78,29 @@ def main ():
     db_access.init(Cnf, projectB);
 
     bin_pkgs = {};
-    miss_src = {};
     src_pkgs = {};
     source_binaries = {};
+    bins_in_suite = {};
+    nbs = {};
+    source_versions = {};
+
+    anais_output = "";
+    duplicate_bins = {};
 
     suite = "unstable";
     suite_id = db_access.get_suite_id(suite);
 
+    bin_not_built = {};
+
+    # Initalize a large hash table of all binary packages
+    before = time.time();
+    sys.stderr.write("[Getting a list of binary packages in %s..." % (suite));
+    q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = 5 AND ba.bin = b.id");
+    ql = q.getresult();
+    sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)));
+    for i in ql:
+        bins_in_suite[i[0]] = "";
+
     components = Cnf.ValueList("Suite::%s::Components" % (suite));
     for component in components:
         filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component);
@@ -88,23 +116,28 @@ def main ():
         Sources = apt_pkg.ParseTagFile(sources);
         while Sources.Step():
             source = Sources.Section.Find('Package');
+            source_version = Sources.Section.Find('Version');
             architecture = Sources.Section.Find('Architecture');
             binaries = Sources.Section.Find('Binary');
+            binaries_list = map(string.strip, binaries.split(','));
+
+            # Check for binaries not built on any architecture.
+            for binary in binaries_list:
+                if not bins_in_suite.has_key(binary):
+                    if not bin_not_built.has_key(source):
+                        bin_not_built[source] = {};
+                    bin_not_built[source][binary] = "";
 
             # Check for packages built on architectures they shouldn't be.
             if architecture != "any" and architecture != "all":
                 architectures = {};
                 for arch in architecture.split():
                     architectures[arch.strip()] = "";
-                for binary in binaries.split(','):
-                    binary = binary.strip();
+                for binary in binaries_list:
                     q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary));
                     ql = q.getresult();
-                    if not ql:
-                        utils.warn("%s lists %s as a binary, but it doesn't seem to exist in %s?" % (source, binary, suite));
-                    # Loop around twice; first to get the latest 'valid' version
                     versions = [];
-                    for i in q.getresult():
+                    for i in ql:
                         arch = i[0];
                         version = i[1];
                         if architectures.has_key(arch):
@@ -114,32 +147,39 @@ def main ():
                         latest_version = versions.pop()
                     else:
                         latest_version = None;
-                    # ... then to check for 'invalid' architectures
-                    for i in q.getresult():
+                    # Check for 'invalid' architectures
+                    versions_d = {}
+                    for i in ql:
                         arch = i[0];
                         version = i[1];
                         if not architectures.has_key(arch):
-                            print "[%s]: %s appears for %s (vs. '%s')" % (source, binary, arch, architecture),
-                            if not latest_version:
-                                print "** mwaap, mwapp!  Ignore me **";
-                                continue;
-                            if apt_pkg.VersionCompare(latest_version, version) != -1:
-                                print "- out of date.",
-                            else:
-                                print "- current.",
-                            print "[%s vs %s (%s)]" % (latest_version, version, arch);
+                            if not versions_d.has_key(version):
+                                versions_d[version] = [];
+                            versions_d[version].append(arch)
+
+                    if versions_d != {}:
+                        anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture);
+                        versions = versions_d.keys();
+                        versions.sort(apt_pkg.VersionCompare);
+                        for version in versions:
+                            arches = versions_d[version];
+                            arches.sort();
+                            anais_output += "    o %s: %s\n" % (version, ", ".join(arches));
 
             # Check for duplicated packages and build indices for checking "no source" later
             source_index = component + '/' + source;
             if src_pkgs.has_key(source):
                 print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]);
             src_pkgs[source] = source_index;
-            for binary in binaries.split(','):
-                binary = binary.strip();
+            for binary in binaries_list:
                 if bin_pkgs.has_key(binary):
-                    print " binary %s is duplicated in source packages %s and %s" % (binary, source, bin_pkgs[binary]);
+                    key = "%s~%s" % (source, bin_pkgs[binary]);
+                    if not duplicate_bins.has_key(key):
+                        duplicate_bins[key] = [];
+                    duplicate_bins[key].append(binary);
                 bin_pkgs[binary] = source;
             source_binaries[source] = binaries;
+            source_versions[source] = source_version;
 
         sources.close();
         os.unlink(temp_filename);
@@ -153,21 +193,35 @@ def main ():
             while Packages.Step():
                 package = Packages.Section.Find('Package');
                 source = Packages.Section.Find('Source', "");
+                version = Packages.Section.Find('Version');
                 if source == "":
                     source = package;
                 if source.find("(") != -1:
-                    m = utils.re_extract_src_version.match(source)
-                    source = m.group(1)
-                if not bin_pkgs.has_key(package) and not miss_src.has_key(package):
-                    miss_src[package] = 1;
-                    print " %s has no source [%s: %s]" % (package, source, source_binaries.get(source, "(source does not exist)"));
+                    m = utils.re_extract_src_version.match(source);
+                    source = m.group(1);
+                    version = m.group(2);
+                if not bin_pkgs.has_key(package):
+                    if not nbs.has_key(source):
+                        nbs[source] = {};
+                    if not nbs[source].has_key(package):
+                        nbs[source][package] = {};
+                    nbs[source][package][version] = "";
             packages.close();
 
-    # Check for packages in experimental obsoleted by versions in unstable
-    #
-    # [If melanie was callable from python, we could auto-remove these
-    #  packages...]
+    dubious_nbs = {};
+    real_nbs = {};
+    for source in nbs.keys():
+        for package in nbs[source].keys():
+            versions = nbs[source][package].keys();
+            versions.sort(apt_pkg.VersionCompare);
+            latest_version = versions.pop();
+            source_version = source_versions.get(source,"0");
+            if apt_pkg.VersionCompare(latest_version, source_version) == 0:
+                add_nbs(dubious_nbs, source, latest_version, package);
+            else:
+                add_nbs(real_nbs, source, latest_version, package);
 
+    # Check for packages in experimental obsoleted by versions in unstable
     suite_id = db_access.get_suite_id("unstable");
     q = projectB.query("""
 SELECT s.source, s.version AS experimental, s2.version AS unstable
@@ -177,8 +231,99 @@ SELECT s.source, s.version AS experimental, s2.version AS unstable
    AND versioncmp(s.version, s2.version) < 0""" % (suite_id));
     ql = q.getresult();
     if ql:
+        nviu_to_remove = [];
+        print "Newer version in unstable";
+        print "-------------------------";
+        print ;
+        for i in ql:
+            (source, experimental_version, unstable_version) = i;
+            print " o %s (%s, %s)" % (source, experimental_version, unstable_version);
+            nviu_to_remove.extend(source);
+        print
+        print "Suggested command:"
+        print " melanie -m \"[rene] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove));
         print
-        print q
+
+    print "Not Built from Source";
+    print "---------------------";
+    print ;
+
+    nbs_to_remove = [];
+    nbs_keys = real_nbs.keys();
+    nbs_keys.sort();
+    for source in nbs_keys:
+        binaries = source_binaries.get(source, "(source does not exist)")
+        print " * %s_%s builds: %s" % (source,
+                                       source_versions.get(source, "??"),
+                                       source_binaries.get(source, "(source does not exist)"));
+        print "      but no longer builds:"
+        versions = real_nbs[source].keys();
+        versions.sort(apt_pkg.VersionCompare);
+        for version in versions:
+            packages = real_nbs[source][version].keys();
+            packages.sort();
+            for pkg in packages:
+                # *cough* FIXME
+                if pkg.find("pcmcia") == -1:
+                    nbs_to_remove.append(pkg);
+            print "        o %s: %s" % (version, ", ".join(packages));
+
+        print ;
+
+    print "Suggested command:"
+    print " melanie -m \"[rene] NBS\" -b %s" % (" ".join(nbs_to_remove));
+    print
+
+    print "="*75
+    print
+
+    print "Unbuilt binary packages";
+    print "-----------------------";
+    print
+    keys = bin_not_built.keys();
+    keys.sort();
+    for source in keys:
+        binaries = bin_not_built[source].keys();
+        binaries.sort();
+        print " o %s: %s" % (source, ", ".join(binaries));
+    print ;
+
+    print "Built from multiple source packages";
+    print "-----------------------------------";
+    print ;
+    keys = duplicate_bins.keys();
+    keys.sort();
+    for key in keys:
+        (source_a, source_b) = key.split("~");
+        print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key]));
+    print ;
+
+    print "Architecture Not Allowed In Source";
+    print "----------------------------------";
+    print anais_output;
+    print ;
+
+    print "Dubious NBS";
+    print "-----------";
+    print ;
+
+    dubious_nbs_keys = dubious_nbs.keys();
+    dubious_nbs_keys.sort();
+    for source in dubious_nbs_keys:
+        binaries = source_binaries.get(source, "(source does not exist)")
+        print " * %s_%s builds: %s" % (source,
+                                       source_versions.get(source, "??"),
+                                       source_binaries.get(source, "(source does not exist)"));
+        print "      won't admit to building:"
+        versions = dubious_nbs[source].keys();
+        versions.sort(apt_pkg.VersionCompare);
+        for version in versions:
+            packages = dubious_nbs[source][version].keys();
+            packages.sort();
+            print "        o %s: %s" % (version, ", ".join(packages));
+
+        print ;
+
 
 ################################################################################