Merge branch 'multiprocessing' into p-s-from-db

[dak.git] / dak / generate_packages_sources2.py

diff --git a/dak/generate_packages_sources2.py b/dak/generate_packages_sources2.py
index 5a1008c424286473ccb943e8012b717b340d9393..3b28c53bde798257baccdea6e226ca01537656e7 100755 (executable)
--- a/dak/generate_packages_sources2.py
+++ b/dak/generate_packages_sources2.py
@@ -31,11 +31,9 @@ Generate Packages/Sources files
 from daklib.dbconn import *
 from daklib.config import Config
 from daklib import utils, daklog
-from multiprocessing import Pool
+from daklib.dakmultiprocessing import Pool
 import apt_pkg, os, stat, sys
 
-from daklib.lists import getSources, getBinaries, getArchAll
-
 def usage():
     print """Usage: dak generate-packages-sources2 [OPTIONS]
 Generate the Packages/Sources files
@@ -65,7 +63,7 @@ SELECT
          WHEN key = 'Checksums-Sha1' THEN E'Checksums-Sha1\:\n ' || f.sha1sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
          WHEN key = 'Checksums-Sha256' THEN E'Checksums-Sha256\:\n ' || f.sha256sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
          ELSE key || '\: '
-       END || value, E'\n' ORDER BY mk.order)
+       END || value, E'\n' ORDER BY mk.ordering, mk.key)
    FROM
      source_metadata sm
      JOIN metadata_keys mk ON mk.key_id = sm.key_id
@@ -81,13 +79,15 @@ SELECT
 FROM
 
 source s
+JOIN src_associations sa ON s.id = sa.source
 JOIN files f ON s.file=f.id
 JOIN override o ON o.package = s.source
 JOIN section sec ON o.section = sec.id
 JOIN priority pri ON o.priority = pri.id
 
 WHERE
-o.suite = :suite AND o.component = :component AND o.type = :dsc_type
+  sa.suite = :suite
+  AND o.suite = :suite AND o.component = :component AND o.type = :dsc_type
 
 ORDER BY
 s.source, s.version
@@ -127,7 +127,9 @@ def generate_sources(suite_id, component_id):
         print >>output, stanza
         print >>output, ""
 
-    return ["generate sources", suite.suite_name, component.component_name]
+    message = ["generate sources", suite.suite_name, component.component_name]
+    session.rollback()
+    return message
 
 #############################################################################
 
@@ -155,14 +157,14 @@ WITH
       JOIN location l ON l.id = f.location
       JOIN source s ON b.source = s.id
     WHERE
-      (b.architecture = :arch_all OR b.architecture = :all) AND b.type = :type_name
+      (b.architecture = :arch_all OR b.architecture = :arch) AND b.type = :type_name
       AND ba.suite = :suite
       AND l.component = :component
   )
 
 SELECT
   (SELECT
-     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY mk.order)
+     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY mk.ordering, mk.key)
    FROM
      binaries_metadata bm
      JOIN metadata_keys mk ON mk.key_id = bm.key_id
@@ -172,7 +174,7 @@ SELECT
   )
   || E'\nSection\: ' || sec.section
   || E'\nPriority\: ' || pri.priority
-  || E'\nFilename\: ' || tmp.filename
+  || E'\nFilename\: pool/' || tmp.filename
   || E'\nSize\: ' || tmp.size
   || E'\nMD5sum\: ' || tmp.md5sum
   || E'\nSHA1\: ' || tmp.sha1sum
@@ -194,6 +196,8 @@ WHERE
   )
   AND
     o.type = :type_id AND o.suite = :suite AND o.component = :component
+
+ORDER BY tmp.package, tmp.version
 """
 
 def open_packages(suite, component, architecture, type_name):
@@ -230,13 +234,14 @@ def generate_packages(suite_id, component_id, architecture_id, type_name):
     output = open_packages(suite, component, architecture, type_name)
 
     r = session.execute(_packages_query, {"suite": suite_id, "component": component_id,
-        "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id})
+        "arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id})
     for (stanza,) in r:
         print >>output, stanza
+        print >>output, ""
 
-    session.close()
-
-    return ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
+    message = ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
+    session.rollback()
+    return message
 
 #############################################################################
 
@@ -272,6 +277,8 @@ def main():
     else:
         suites = session.query(Suite).filter(Suite.untouchable == False).all()
 
+    force = Options.has_key("Force") and Options["Force"]
+
     component_ids = [ c.component_id for c in session.query(Component).all() ]
 
     def log(details):
@@ -279,12 +286,13 @@ def main():
 
     pool = Pool()
     for s in suites:
+        if s.untouchable and not force:
+            utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
         for c in component_ids:
             pool.apply_async(generate_sources, [s.suite_id, c], callback=log)
             for a in s.architectures:
-                #pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=log)
-                apply(generate_packages, [s.suite_id, c, a.arch_id, 'deb'])
-                #pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=log)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=log)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=log)
 
     pool.close()
     pool.join()