daklib/queue.py: Remove unneeded imports.

[dak.git] / dak / generate_filelist.py

diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py
index dcf6864ff72412e98b3797b908eb49213e70de74..faf08bd7a8680e59802d4bdf8c12d4ec0a68a21b 100755 (executable)
--- a/dak/generate_filelist.py
+++ b/dak/generate_filelist.py
@@ -5,6 +5,7 @@ Generate file lists for apt-ftparchive.
 
 @contact: Debian FTP Master <ftpmaster@debian.org>
 @copyright: 2009  Torsten Werner <twerner@debian.org>
+@copyright: 2011  Ansgar Burchardt <ansgar@debian.org>
 @license: GNU General Public License version 2 or later
 """
 
@@ -37,11 +38,11 @@ Generate file lists for apt-ftparchive.
 
 from daklib.dbconn import *
 from daklib.config import Config
-from daklib.threadpool import ThreadPool
-from daklib import utils
+from daklib import utils, daklog
+from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
 import apt_pkg, os, stat, sys
 
-from daklib.lists import getSources, getBinaries
+from daklib.lists import getSources, getBinaries, getArchAll
 
 def listPath(suite, component, architecture = None, type = None,
         incremental_mode = False):
@@ -64,26 +65,54 @@ def listPath(suite, component, architecture = None, type = None,
         file.truncate()
     return (file, timestamp)
 
-def writeSourceList(args):
-    (suite, component, incremental_mode) = args
+def writeSourceList(suite_id, component_id, incremental_mode):
+    session = DBConn().session()
+    suite = Suite.get(suite_id, session)
+    component = Component.get(component_id, session)
     (file, timestamp) = listPath(suite, component,
             incremental_mode = incremental_mode)
-    session = DBConn().session()
+
+    message = "sources list for %s %s" % (suite.suite_name, component.component_name)
+
     for _, filename in getSources(suite, component, session, timestamp):
         file.write(filename + '\n')
-    session.close()
+    session.rollback()
     file.close()
+    return (PROC_STATUS_SUCCESS, message)
 
-def writeBinaryList(args):
-    (suite, component, architecture, type, incremental_mode) = args
+def writeAllList(suite_id, component_id, architecture_id, type, incremental_mode):
+    session = DBConn().session()
+    suite = Suite.get(suite_id, session)
+    component = Component.get(component_id, session)
+    architecture = Architecture.get(architecture_id, session)
     (file, timestamp) = listPath(suite, component, architecture, type,
             incremental_mode)
+
+    message = "all list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type)
+
+    for _, filename in getArchAll(suite, component, architecture, type,
+            session, timestamp):
+        file.write(filename + '\n')
+    session.rollback()
+    file.close()
+    return (PROC_STATUS_SUCCESS, message)
+
+def writeBinaryList(suite_id, component_id, architecture_id, type, incremental_mode):
     session = DBConn().session()
+    suite = Suite.get(suite_id, session)
+    component = Component.get(component_id, session)
+    architecture = Architecture.get(architecture_id, session)
+    (file, timestamp) = listPath(suite, component, architecture, type,
+            incremental_mode)
+
+    message = "binary list for %s %s (arch=%s, type=%s)" % (suite.suite_name, component.component_name, architecture.arch_string, type)
+
     for _, filename in getBinaries(suite, component, architecture, type,
             session, timestamp):
         file.write(filename + '\n')
-    session.close()
+    session.rollback()
     file.close()
+    return (PROC_STATUS_SUCCESS, message)
 
 def usage():
     print """Usage: dak generate_filelist [OPTIONS]
@@ -103,6 +132,7 @@ Incremental mode appends only newer files to existing lists."""
 
 def main():
     cnf = Config()
+    Logger = daklog.Logger('generate-filelist')
     Arguments = [('h', "help",         "Filelist::Options::Help"),
                  ('s', "suite",        "Filelist::Options::Suite", "HasArg"),
                  ('c', "component",    "Filelist::Options::Component", "HasArg"),
@@ -125,36 +155,63 @@ def main():
         cnf['Filelist::Options::Architecture'] = ','.join(architectures).encode()
     cnf['Filelist::Options::Help'] = ''
     cnf['Filelist::Options::Incremental'] = ''
-    apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
-    Options = cnf.SubTree("Filelist::Options")
+    apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
+    Options = cnf.subtree("Filelist::Options")
     if Options['Help']:
         usage()
-    threadpool = ThreadPool()
+    pool = DakProcessPool()
     query_suites = query_suites. \
         filter(Suite.suite_name.in_(utils.split_args(Options['Suite'])))
     query_components = query_components. \
         filter(Component.component_name.in_(utils.split_args(Options['Component'])))
     query_architectures = query_architectures. \
         filter(Architecture.arch_string.in_(utils.split_args(Options['Architecture'])))
+
+    def parse_results(message):
+        # Split out into (code, msg)
+        code, msg = message
+        if code == PROC_STATUS_SUCCESS:
+            Logger.log([msg])
+        elif code == PROC_STATUS_SIGNALRAISED:
+            Logger.log(['E: Subprocess recieved signal ', msg])
+        else:
+            Logger.log(['E: ', msg])
+
     for suite in query_suites:
+        suite_id = suite.suite_id
         for component in query_components:
+            component_id = component.component_id
             for architecture in query_architectures:
+                architecture_id = architecture.arch_id
                 if architecture not in suite.architectures:
                     pass
                 elif architecture.arch_string == 'source':
-                    threadpool.queueTask(writeSourceList,
-                        (suite, component, Options['Incremental']))
-                elif architecture.arch_string != 'all':
-                    threadpool.queueTask(writeBinaryList,
-                        (suite, component, architecture, 'deb',
-                            Options['Incremental']))
-                    threadpool.queueTask(writeBinaryList,
-                        (suite, component, architecture, 'udeb',
-                            Options['Incremental']))
-    threadpool.joinAll()
+                    pool.apply_async(writeSourceList,
+                        (suite_id, component_id, Options['Incremental']), callback=parse_results)
+                elif architecture.arch_string == 'all':
+                    pool.apply_async(writeAllList,
+                        (suite_id, component_id, architecture_id, 'deb',
+                            Options['Incremental']), callback=parse_results)
+                    pool.apply_async(writeAllList,
+                        (suite_id, component_id, architecture_id, 'udeb',
+                            Options['Incremental']), callback=parse_results)
+                else: # arch any
+                    pool.apply_async(writeBinaryList,
+                        (suite_id, component_id, architecture_id, 'deb',
+                            Options['Incremental']), callback=parse_results)
+                    pool.apply_async(writeBinaryList,
+                        (suite_id, component_id, architecture_id, 'udeb',
+                            Options['Incremental']), callback=parse_results)
+    pool.close()
+    pool.join()
+
     # this script doesn't change the database
     session.close()
 
+    Logger.close()
+
+    sys.exit(pool.overall_status())
+
 if __name__ == '__main__':
     main()