X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=0f1125f81bc3c8f7eba5dcf9b170a81449ee0a9e;hb=af486e867c2809515c09ae4f854a95355112010e;hp=6837ebf7cd2c8e22f82c32a0cd2b7c6408a7a5b7;hpb=774a27caff1cc21340ee1b08528512e96a0eed5b;p=dak.git

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 6837ebf7..0f1125f8 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -27,7 +27,7 @@
 
 ################################################################################
 
-import commands, pg, os, sys, time
+import commands, pg, os, sys, time, re
 import apt_pkg
 from daklib import database
 from daklib import utils
@@ -51,7 +51,8 @@ Check for obsolete or duplicated packages.
 
   -h, --help                show this help and exit.
   -m, --mode=MODE           chose the MODE to run in (full or daily).
-  -s, --suite=SUITE         check suite SUITE."""
+  -s, --suite=SUITE         check suite SUITE.
+  -w, --wanna-build-dump    where to find the copies of http://buildd.debian.org/stats/*.txt"""
     sys.exit(exit_code)
 
 ################################################################################
@@ -114,6 +115,61 @@ def do_anais(architecture, binaries_list, source):
                 anais_output += "    o %s: %s\n" % (version, ", ".join(arches))
     return anais_output
 
+
+################################################################################
+
+# Check for out-of-date binaries on architectures that do not want to build that
+# package any more, and have them listed as Not-For-Us
+def do_nfu(nfu_packages):
+    output = ""
+    
+    a2p = {}
+
+    for architecture in nfu_packages:
+        a2p[architecture] = []
+        for (package,bver,sver) in nfu_packages[architecture]:
+            output += "  * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver)
+            a2p[architecture].append(package)
+
+
+    if output:
+        print "Obsolete by Not-For-Us"
+        print "----------------------"
+        print
+        print output
+
+        print "Suggested commands:"
+        for architecture in a2p:
+            if a2p[architecture]:
+                print (" dak rm -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % 
+                    (suite, architecture, " ".join(a2p[architecture])))
+        print
+
+def parse_nfu(architecture):
+    # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
+    r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
+
+    ret = set()
+    
+    filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
+
+    # Not all architectures may have a wanna-build dump, so we want to ignore missin
+    # files
+    if os.path.exists(filename):
+        f = utils.open_file(filename)
+        for line in f:
+            if line[0] == ' ':
+                continue
+
+            m = r.match(line)
+            if m:
+                ret.add(m.group(1))
+
+        f.close()
+    else:
+        utils.warn("No wanna-build dump file for architecture %s", architecture)
+    return ret
+
 ################################################################################
 
 def do_nviu():
@@ -126,8 +182,8 @@ SELECT s.source, s.version AS experimental, s2.version AS unstable
   FROM src_associations sa, source s, source s2, src_associations sa2
   WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id
    AND sa2.source = s2.id AND s.source = s2.source
-   AND versioncmp(s.version, s2.version) < 0""" % (experimental_id,
-                                                   database.get_suite_id("unstable")))
+   AND s.version < s2.version""" % (experimental_id,
+                                    database.get_suite_id("unstable")))
     ql = q.getresult()
     if ql:
         nviu_to_remove = []
@@ -265,7 +321,8 @@ def main ():
 
     Arguments = [('h',"help","Cruft-Report::Options::Help"),
                  ('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
-                 ('s',"suite","Cruft-Report::Options::Suite","HasArg")]
+                 ('s',"suite","Cruft-Report::Options::Suite","HasArg"),
+                 ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
     for i in [ "help" ]:
         if not Cnf.has_key("Cruft-Report::Options::%s" % (i)):
             Cnf["Cruft-Report::Options::%s" % (i)] = ""
@@ -274,6 +331,9 @@ def main ():
     if not Cnf.has_key("Cruft-Report::Options::Mode"):
         Cnf["Cruft-Report::Options::Mode"] = "daily"
 
+    if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
+        Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu"
+
     apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
 
     Options = Cnf.SubTree("Cruft-Report::Options")
@@ -284,7 +344,7 @@ def main ():
     if Options["Mode"] == "daily":
         checks = [ "nbs", "nviu", "obsolete source" ]
     elif Options["Mode"] == "full":
-        checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ]
+        checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
     else:
         utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"]))
         usage(1)
@@ -302,6 +362,8 @@ def main ():
     anais_output = ""
     duplicate_bins = {}
 
+    nfu_packages = {}
+
     suite = Options["Suite"]
     suite_id = database.get_suite_id(suite)
 
@@ -315,7 +377,7 @@ def main ():
     for component in components:
         filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component)
         # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance...
-        temp_filename = utils.temp_filename()
+        (fd, temp_filename) = utils.temp_filename()
         (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
         if (result != 0):
             sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
@@ -367,11 +429,16 @@ def main ():
         for architecture in architectures:
             filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture)
             # apt_pkg.ParseTagFile needs a real file handle
-            temp_filename = utils.temp_filename()
+            (fd, temp_filename) = utils.temp_filename()
             (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
             if (result != 0):
                 sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
                 sys.exit(result)
+
+            if "nfu" in checks:
+                nfu_packages.setdefault(architecture,[])
+                nfu_entries = parse_nfu(architecture)
+
             packages = utils.open_file(temp_filename)
             Packages = apt_pkg.ParseTagFile(packages)
             while Packages.Step():
@@ -405,6 +472,11 @@ def main ():
                         duplicate_bins.setdefault(key, [])
                         if package not in duplicate_bins[key]:
                             duplicate_bins[key].append(package)
+                    if "nfu" in checks:
+                        if package in nfu_entries and \
+                               version != source_versions[source]: # only suggest to remove out-of-date packages
+                            nfu_packages[architecture].append((package,version,source_versions[source]))
+                    
             packages.close()
             os.unlink(temp_filename)
 
@@ -437,6 +509,9 @@ def main ():
         print "="*75
         print
 
+    if "nfu" in checks:
+        do_nfu(nfu_packages)
+
     if "bnb" in checks:
         print "Unbuilt binary packages"
         print "-----------------------"