Merge remote branch 'ftpmaster/master' into multiproc

[dak.git] / dak / generate_packages_sources2.py

diff --git a/dak/generate_packages_sources2.py b/dak/generate_packages_sources2.py
index ea3c0e53b21866ab40dcff27f75171121d6fb341..b157fcbe5fc989d0bb1adce55b9237c1d11313e1 100755 (executable)
--- a/dak/generate_packages_sources2.py
+++ b/dak/generate_packages_sources2.py
@@ -31,7 +31,7 @@ Generate Packages/Sources files
 from daklib.dbconn import *
 from daklib.config import Config
 from daklib import utils, daklog
-from daklib.dakmultiprocessing import Pool
+from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
 from daklib.filewriter import PackagesFileWriter, SourcesFileWriter
 
 import apt_pkg, os, stat, sys
@@ -89,7 +89,7 @@ JOIN priority pri ON o.priority = pri.id
 
 WHERE
   sa.suite = :suite
-  AND o.suite = :suite AND o.component = :component AND o.type = :dsc_type
+  AND o.suite = :overridesuite AND o.component = :component AND o.type = :dsc_type
 
 ORDER BY
 s.source, s.version
@@ -104,11 +104,13 @@ def generate_sources(suite_id, component_id):
     suite = session.query(Suite).get(suite_id)
     component = session.query(Component).get(component_id)
 
+    overridesuite_id = suite.get_overridesuite().suite_id
+
     writer = SourcesFileWriter(suite=suite.suite_name, component=component.component_name)
     output = writer.open()
 
     # run query and write Sources
-    r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "dsc_type": dsc_type})
+    r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "dsc_type": dsc_type, "overridesuite": overridesuite_id})
     for (stanza,) in r:
         print >>output, stanza
         print >>output, ""
@@ -117,7 +119,7 @@ def generate_sources(suite_id, component_id):
 
     message = ["generate sources", suite.suite_name, component.component_name]
     session.rollback()
-    return message
+    return (PROC_STATUS_SUCCESS, message)
 
 #############################################################################
 
@@ -160,6 +162,13 @@ SELECT
      bm.bin_id = tmp.binary_id
      AND key != 'Section' AND key != 'Priority'
   )
+  || COALESCE(E'\n' || (SELECT
+     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY key)
+   FROM external_overrides eo
+   WHERE
+     eo.package = tmp.package
+     AND eo.suite = :overridesuite AND eo.component = :component
+  ), '')
   || E'\nSection\: ' || sec.section
   || E'\nPriority\: ' || pri.priority
   || E'\nFilename\: pool/' || tmp.filename
@@ -183,7 +192,7 @@ WHERE
       (architecture = :arch_all AND source NOT IN (SELECT DISTINCT source FROM tmp WHERE architecture <> :arch_all))
   )
   AND
-    o.type = :type_id AND o.suite = :suite AND o.component = :component
+    o.type = :type_id AND o.suite = :overridesuite AND o.component = :component
 
 ORDER BY tmp.package, tmp.version
 """
@@ -199,12 +208,15 @@ def generate_packages(suite_id, component_id, architecture_id, type_name):
     component = session.query(Component).get(component_id)
     architecture = session.query(Architecture).get(architecture_id)
 
+    overridesuite_id = suite.get_overridesuite().suite_id
+
     writer = PackagesFileWriter(suite=suite.suite_name, component=component.component_name,
             architecture=architecture.arch_string, debtype=type_name)
     output = writer.open()
 
     r = session.execute(_packages_query, {"suite": suite_id, "component": component_id,
-        "arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id})
+        "arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id,
+        "overridesuite": overridesuite_id})
     for (stanza,) in r:
         print >>output, stanza
         print >>output, ""
@@ -213,7 +225,7 @@ def generate_packages(suite_id, component_id, architecture_id, type_name):
 
     message = ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
     session.rollback()
-    return message
+    return (PROC_STATUS_SUCCESS, message)
 
 #############################################################################
 
@@ -253,23 +265,35 @@ def main():
 
     component_ids = [ c.component_id for c in session.query(Component).all() ]
 
-    def log(details):
-        logger.log(details)
-
-    pool = Pool()
+    def parse_results(message):
+        # Split out into (code, msg)
+        code, msg = message
+        if code == PROC_STATUS_SUCCESS:
+            logger.log([msg])
+        elif code == PROC_STATUS_SIGNALRAISED:
+            logger.log(['E: Subprocess recieved signal ', msg])
+        else:
+            logger.log(['E: ', msg])
+
+    pool = DakProcessPool()
     for s in suites:
         if s.untouchable and not force:
             utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
         for c in component_ids:
-            pool.apply_async(generate_sources, [s.suite_id, c], callback=log)
+            pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
             for a in s.architectures:
-                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=log)
-                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=log)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)
 
     pool.close()
     pool.join()
+
     # this script doesn't change the database
     session.close()
 
+    logger.close()
+
+    sys.exit(pool.overall_status())
+
 if __name__ == '__main__':
     main()