X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=fab47bf056d72bf910fd774ed48c45e584487ce6;hb=d80d1f9473ed63a08404a23c04a9d8eabedc76a6;hp=238cdd497463dbea6824437b4faa994940619eb5;hpb=5819288346c6ea5d9b23125586969cbec3bc6d44;p=dak.git

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 238cdd49..fab47bf0 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -27,10 +27,10 @@
 
 ################################################################################
 
-import commands, pg, os, sys, time
+import commands, pg, os, sys, time, re
 import apt_pkg
-import daklib.database as database
-import daklib.utils as utils
+from daklib import database
+from daklib import utils
 
 ################################################################################
 
@@ -51,7 +51,8 @@ Check for obsolete or duplicated packages.
 
   -h, --help                show this help and exit.
   -m, --mode=MODE           chose the MODE to run in (full or daily).
-  -s, --suite=SUITE         check suite SUITE."""
+  -s, --suite=SUITE         check suite SUITE.
+  -w, --wanna-build-dump    where to find the copies of http://buildd.debian.org/stats/*.txt"""
     sys.exit(exit_code)
 
 ################################################################################
@@ -114,6 +115,61 @@ def do_anais(architecture, binaries_list, source):
                 anais_output += "    o %s: %s\n" % (version, ", ".join(arches))
     return anais_output
 
+
+################################################################################
+
+# Check for out-of-date binaries on architectures that do not want to build that
+# package any more, and have them listed as Not-For-Us
+def do_nfu(nfu_packages):
+    output = ""
+    
+    a2p = {}
+
+    for architecture in nfu_packages:
+        a2p[architecture] = []
+        for (package,bver,sver) in nfu_packages[architecture]:
+            output += "  * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver)
+            a2p[architecture].append(package)
+
+
+    if output:
+        print "Obsolete by Not-For-Us"
+        print "----------------------"
+        print
+        print output
+
+        print "Suggested commands:"
+        for architecture in a2p:
+            if a2p[architecture]:
+                print (" dak rm -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % 
+                    (suite, architecture, " ".join(a2p[architecture])))
+        print
+
+def parse_nfu(architecture):
+    # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
+    r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
+
+    ret = set()
+    
+    filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
+
+    # Not all architectures may have a wanna-build dump, so we want to ignore missin
+    # files
+    if os.path.exists(filename):
+        f = utils.open_file(filename)
+        for line in f:
+            if line[0] == ' ':
+                continue
+
+            m = r.match(line)
+            if m:
+                ret.add(m.group(1))
+
+        f.close()
+    else:
+        utils.warn("No wanna-build dump file for architecture %s", architecture)
+    return ret
+
 ################################################################################
 
 def do_nviu():
@@ -242,6 +298,20 @@ def do_obsolete_source(duplicate_bins, bin2source):
         print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
         print
 
+def get_suite_binaries():
+    # Initalize a large hash table of all binary packages
+    binaries = {}
+    before = time.time()
+
+    sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
+    q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
+    ql = q.getresult()
+    sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
+    for i in ql:
+        binaries[i[0]] = ""
+
+    return binaries
+
 ################################################################################
 
 def main ():
@@ -251,7 +321,8 @@ def main ():
 
     Arguments = [('h',"help","Cruft-Report::Options::Help"),
                  ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
-                 ('s',"suite","Cruft-Report::Options::Suite","HasArg")]
+                 ('s',"suite","Cruft-Report::Options::Suite","HasArg"),
+                 ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
     for i in [ "help" ]:
         if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
             Cnf["Cruft-Report::Options::%s" % (i)] = ""
@@ -260,6 +331,9 @@ def main ():
     if not Cnf.has_key("Cruft-Report::Options::Mode"):
         Cnf["Cruft-Report::Options::Mode"] = "daily"
 
+    if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
+        Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
+
     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 
     Options = Cnf.SubTree("Cruft-Report::Options")
@@ -270,7 +344,7 @@ def main ():
     if Options["Mode"] == "daily":
         checks = [ "nbs", "nviu", "obsolete source" ]
     elif Options["Mode"] == "full":
-        checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]
+        checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
     else:
         utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
         usage(1)
@@ -288,20 +362,15 @@ def main ():
     anais_output = ""
     duplicate_bins = {}
 
+    nfu_packages = {}
+
     suite = Options["Suite"]
     suite_id = database.get_suite_id(suite)
 
     bin_not_built = {}
 
     if "bnb" in checks:
-        # Initalize a large hash table of all binary packages
-        before = time.time()
-        sys.stderr.write("[Getting a list of binary packages in %s..." % (suite))
-        q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id))
-        ql = q.getresult()
-        sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before)))
-        for i in ql:
-            bins_in_suite[i[0]] = ""
+        bins_in_suite = get_suite_binaries()
 
     # Checks based on the Sources files
     components = Cnf.ValueList("Suite::%s::Components" % (suite))
@@ -365,6 +434,11 @@ def main ():
             if (result != 0):
                 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
                 sys.exit(result)
+
+            if "nfu" in checks:
+                nfu_packages.setdefault(architecture,[])
+                nfu_entries = parse_nfu(architecture)
+
             packages = utils.open_file(temp_filename)
             Packages = apt_pkg.ParseTagFile(packages)
             while Packages.Step():
@@ -398,6 +472,11 @@ def main ():
                         duplicate_bins.setdefault(key, [])
                         if package not in duplicate_bins[key]:
                             duplicate_bins[key].append(package)
+                    if "nfu" in checks:
+                        if package in nfu_entries and \
+                               version != source_versions[source]: # only suggest to remove out-of-date packages
+                            nfu_packages[architecture].append((package,version,source_versions[source]))
+                    
             packages.close()
             os.unlink(temp_filename)
 
@@ -430,6 +509,9 @@ def main ():
         print "="*75
         print
 
+    if "nfu" in checks:
+        do_nfu(nfu_packages)
+
     if "bnb" in checks:
         print "Unbuilt binary packages"
         print "-----------------------"