gps timeout/error handling

[dak.git] / dak / generate_packages_sources.py

diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py
index 7398f67078bf37f6e6b31d0ab647f9ff6dfbf5eb..07a64b6e823ee02610be52fecd34e0a8114bc40e 100755 (executable)
--- a/dak/generate_packages_sources.py
+++ b/dak/generate_packages_sources.py
@@ -31,7 +31,7 @@ import sys
 import apt_pkg
 from tempfile import mkstemp, mkdtemp
 import commands
-from multiprocessing import Pool
+from multiprocessing import Pool, TimeoutError
 
 from daklib import daklog
 from daklib.dbconn import *
@@ -111,17 +111,17 @@ tree "dists/testing"
    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
    Sections "main contrib non-free";
    Architectures "%(arch)s";
-   BinOverride "override.squeeze.$(SECTION)";
-   ExtraOverride "override.squeeze.extra.$(SECTION)";
-   SrcOverride "override.squeeze.$(SECTION).src";
+   BinOverride "override.wheezy.$(SECTION)";
+   ExtraOverride "override.wheezy.extra.$(SECTION)";
+   SrcOverride "override.wheezy.$(SECTION).src";
 };
 """
 
-    apt_trees["squeeze-volatile"]="""
-tree "dists/squeeze-volatile"
+    apt_trees["squeeze-updates"]="""
+tree "dists/squeeze-updates"
 {
-   FileList "/srv/ftp-master.debian.org/database/dists/squeeze-volatile_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-volatile_$(SECTION)_source.list";
+   FileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/squeeze-updates_$(SECTION)_source.list";
    Sections "main contrib non-free";
    Architectures "%(arch)s";
    BinOverride "override.squeeze.$(SECTION)";
@@ -137,8 +137,8 @@ tree "dists/testing/main"
    FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
    Sections "debian-installer";
    Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
+   BinOverride "override.wheezy.main.$(SECTION)";
+   SrcOverride "override.wheezy.main.src";
    BinCacheDB "packages-debian-installer-$(ARCH).db";
    Packages::Extensions ".udeb";
    %(contentsline)s
@@ -149,8 +149,8 @@ tree "dists/testing/non-free"
    FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
    Sections "debian-installer";
    Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
+   BinOverride "override.wheezy.main.$(SECTION)";
+   SrcOverride "override.wheezy.main.src";
    BinCacheDB "packages-debian-installer-$(ARCH).db";
    Packages::Extensions ".udeb";
    %(contentsline)s
@@ -239,9 +239,9 @@ tree "dists/testing-proposed-updates"
    SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
    Sections "main contrib non-free";
    Architectures "%(arch)s";
-   BinOverride "override.squeeze.$(SECTION)";
-   ExtraOverride "override.squeeze.extra.$(SECTION)";
-   SrcOverride "override.squeeze.$(SECTION).src";
+   BinOverride "override.wheezy.$(SECTION)";
+   ExtraOverride "override.wheezy.extra.$(SECTION)";
+   SrcOverride "override.wheezy.$(SECTION).src";
    Contents " ";
 };
 """
@@ -251,8 +251,8 @@ tree "dists/testing-proposed-updates/main"
    FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
    Sections "debian-installer";
    Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
+   BinOverride "override.wheezy.main.$(SECTION)";
+   SrcOverride "override.wheezy.main.src";
    BinCacheDB "packages-debian-installer-$(ARCH).db";
    Packages::Extensions ".udeb";
    Contents " ";
@@ -266,9 +266,9 @@ tree "dists/proposed-updates"
    SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
    Sections "main contrib non-free";
    Architectures "%(arch)s";
-   BinOverride "override.lenny.$(SECTION)";
-   ExtraOverride "override.lenny.extra.$(SECTION)";
-   SrcOverride "override.lenny.$(SECTION).src";
+   BinOverride "override.squeeze.$(SECTION)";
+   ExtraOverride "override.squeeze.extra.$(SECTION)";
+   SrcOverride "override.squeeze.$(SECTION).src";
    Contents " ";
 };
 """
@@ -278,6 +278,32 @@ tree "dists/proposed-updates/main"
    FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
    Sections "debian-installer";
    Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents " ";
+};
+"""
+    apt_trees["oldstable-proposed-updates"]="""
+tree "dists/oldstable-proposed-updates"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.lenny.$(SECTION)";
+   ExtraOverride "override.lenny.extra.$(SECTION)";
+   SrcOverride "override.lenny.$(SECTION).src";
+   Contents " ";
+};
+"""
+    apt_trees["di"]["oldstable-proposed-updates"]="""
+tree "dists/oldstable-proposed-updates/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/oldstable-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
    BinOverride "override.lenny.main.$(SECTION)";
    SrcOverride "override.lenny.main.src";
    BinCacheDB "packages-debian-installer-$(ARCH).db";
@@ -312,9 +338,10 @@ tree "dists/proposed-updates/main"
         # it has errormessages we like to see
         os.environ['GZIP'] = '--rsyncable'
         os.chdir(tmppath)
-        (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
+        (result, output) = commands.getstatusoutput('apt-ftparchive -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name))
         sn="a-f %s,%s: " % (suite, arch)
         print sn + output.replace('\n', '\n%s' % (sn))
+        return result
 
     # Clean up any left behind files
     finally:
@@ -375,6 +402,7 @@ def main ():
     startdir = os.getcwd()
     os.chdir(cnf["Dir::TempPath"])
 
+    broken=[]
     # For each given suite, each architecture, run one apt-ftparchive
     for s in suites:
         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
@@ -382,17 +410,35 @@ def main ():
         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
         for a in arch_list:
-            pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
+            try:
+                result=pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
+                # Get the result. Should it take too long (a-f hanging), break out.
+                r=result.get(timeout=3600)
+            except TimeoutError:
+                broken.append("Timeout: %s - %s" % (s.suite_name, a.arch_string))
+                # Now try the next architecture
+                continue
+
+            if r:
+                # As long as we get 0, we are fine. Otherwise we yell about it later.
+                broken.append("Breakage: %s - %s returned %s" % (s.suite_name, a.arch_string, r))
 
         # No more work will be added to our pool, close it and then wait for all to finish
         pool.close()
         pool.join()
 
+    if len(broken) > 0:
+        Logger.log(['Trouble: %s' % (broken)])
+        print "Trouble: %s" % (broken)
+
     os.chdir(startdir)
     # this script doesn't change the database
     session.close()
     Logger.close()
 
+    if len(broken) > 0:
+        sys.exit(1)
+
 #######################################################################################
 
 if __name__ == '__main__':