Merge remote-tracking branch 'ansgar/p-s-from-db' into merge

[dak.git] / dak / generate_packages_sources2.py

diff --git a/dak/generate_packages_sources2.py b/dak/generate_packages_sources2.py
index b6bc1c21e4e93f5fba5c259a5fb7847213cb85cb..c23ede9bcd3a2a246e3a1a71d76b3a935a825642 100755 (executable)
--- a/dak/generate_packages_sources2.py
+++ b/dak/generate_packages_sources2.py
@@ -31,7 +31,9 @@ Generate Packages/Sources files
 from daklib.dbconn import *
 from daklib.config import Config
 from daklib import utils, daklog
-from multiprocessing import Pool
+from daklib.dakmultiprocessing import Pool
+from daklib.filewriter import PackagesFileWriter, SourcesFileWriter
+
 import apt_pkg, os, stat, sys
 
 def usage():
@@ -63,7 +65,7 @@ SELECT
          WHEN key = 'Checksums-Sha1' THEN E'Checksums-Sha1\:\n ' || f.sha1sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
          WHEN key = 'Checksums-Sha256' THEN E'Checksums-Sha256\:\n ' || f.sha256sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
          ELSE key || '\: '
-       END || value, E'\n' ORDER BY mk.order, mk.key)
+       END || value, E'\n' ORDER BY mk.ordering, mk.key)
    FROM
      source_metadata sm
      JOIN metadata_keys mk ON mk.key_id = sm.key_id
@@ -93,23 +95,6 @@ ORDER BY
 s.source, s.version
 """
 
-def open_sources(suite, component):
-    cnf = Config()
-    dest = os.path.join(cnf['Dir::Root'], 'dists', suite.suite_name, component.component_name, 'source', 'Sources')
-
-    # create queue if it does not exist yet
-    if os.path.exists(dest) and os.path.isdir(dest):
-        dest_dir = dest
-    else:
-        dest_dir = os.path.dirname(dest)
-    if not os.path.exists(dest_dir):
-        umask = os.umask(00000)
-        os.makedirs(dest_dir, 02775)
-        os.umask(umask)
-
-    f = open(dest, 'w')
-    return f
-
 def generate_sources(suite_id, component_id):
     global _sources_query
 
@@ -119,7 +104,8 @@ def generate_sources(suite_id, component_id):
     suite = session.query(Suite).get(suite_id)
     component = session.query(Component).get(component_id)
 
-    output = open_sources(suite, component)
+    writer = SourcesFileWriter(suite=suite.suite_name, component=component.component_name)
+    output = writer.open()
 
     # run query and write Sources
     r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "dsc_type": dsc_type})
@@ -127,7 +113,11 @@ def generate_sources(suite_id, component_id):
         print >>output, stanza
         print >>output, ""
 
-    return ["generate sources", suite.suite_name, component.component_name]
+    writer.close()
+
+    message = ["generate sources", suite.suite_name, component.component_name]
+    session.rollback()
+    return message
 
 #############################################################################
 
@@ -162,7 +152,7 @@ WITH
 
 SELECT
   (SELECT
-     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY mk.order, mk.key)
+     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY mk.ordering, mk.key)
    FROM
      binaries_metadata bm
      JOIN metadata_keys mk ON mk.key_id = bm.key_id
@@ -170,6 +160,11 @@ SELECT
      bm.bin_id = tmp.binary_id
      AND key != 'Section' AND key != 'Priority'
   )
+  || COALESCE(E'\n' || (SELECT
+     STRING_AGG(key || '\: ' || value, E'\n' ORDER BY key)
+   FROM external_overrides eo
+   WHERE eo.package = tmp.package
+  ), '')
   || E'\nSection\: ' || sec.section
   || E'\nPriority\: ' || pri.priority
   || E'\nFilename\: pool/' || tmp.filename
@@ -198,26 +193,6 @@ WHERE
 ORDER BY tmp.package, tmp.version
 """
 
-def open_packages(suite, component, architecture, type_name):
-    cnf = Config()
-    if type_name == 'udeb':
-        dest = os.path.join(cnf['Dir::Root'], 'dists', suite.suite_name, component.component_name, 'debian-installer', 'binary-%s' % architecture.arch_string, 'Packages')
-    else:
-        dest = os.path.join(cnf['Dir::Root'], 'dists', suite.suite_name, component.component_name, 'binary-%s' % architecture.arch_string, 'Packages')
-
-    # create queue if it does not exist yet
-    if os.path.exists(dest) and os.path.isdir(dest):
-        dest_dir = dest
-    else:
-        dest_dir = os.path.dirname(dest)
-    if not os.path.exists(dest_dir):
-        umask = os.umask(00000)
-        os.makedirs(dest_dir, 02775)
-        os.umask(umask)
-
-    f = open(dest, 'w')
-    return f
-
 def generate_packages(suite_id, component_id, architecture_id, type_name):
     global _packages_query
 
@@ -229,7 +204,9 @@ def generate_packages(suite_id, component_id, architecture_id, type_name):
     component = session.query(Component).get(component_id)
     architecture = session.query(Architecture).get(architecture_id)
 
-    output = open_packages(suite, component, architecture, type_name)
+    writer = PackagesFileWriter(suite=suite.suite_name, component=component.component_name,
+            architecture=architecture.arch_string, debtype=type_name)
+    output = writer.open()
 
     r = session.execute(_packages_query, {"suite": suite_id, "component": component_id,
         "arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id})
@@ -237,9 +214,11 @@ def generate_packages(suite_id, component_id, architecture_id, type_name):
         print >>output, stanza
         print >>output, ""
 
-    session.close()
+    writer.close()
 
-    return ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
+    message = ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
+    session.rollback()
+    return message
 
 #############################################################################
 
@@ -289,9 +268,8 @@ def main():
         for c in component_ids:
             pool.apply_async(generate_sources, [s.suite_id, c], callback=log)
             for a in s.architectures:
-                #pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=log)
-                apply(generate_packages, [s.suite_id, c, a.arch_id, 'deb'])
-                #pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=log)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=log)
+                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=log)
 
     pool.close()
     pool.join()