Merge remote-tracking branch 'origin/master' into build-queues

diff --git a/dak/generate_filelist.py b/dak/generate_filelist.py
index dcf6864ff72412e98b3797b908eb49213e70de74..1f4d665495ed21497b421f2d8ea5d4c7b6080570 100755 (executable)
--- a/dak/generate_filelist.py
+++ b/dak/generate_filelist.py
@@ -41,7 +41,7 @@ from daklib.threadpool import ThreadPool
 from daklib import utils
 import apt_pkg, os, stat, sys
 
-from daklib.lists import getSources, getBinaries
+from daklib.lists import getSources, getBinaries, getArchAll
 
 def listPath(suite, component, architecture = None, type = None,
         incremental_mode = False):
@@ -74,6 +74,17 @@ def writeSourceList(args):
     session.close()
     file.close()
 
+def writeAllList(args):
+    (suite, component, architecture, type, incremental_mode) = args
+    (file, timestamp) = listPath(suite, component, architecture, type,
+            incremental_mode)
+    session = DBConn().session()
+    for _, filename in getArchAll(suite, component, architecture, type,
+            session, timestamp):
+        file.write(filename + '\n')
+    session.close()
+    file.close()
+
 def writeBinaryList(args):
     (suite, component, architecture, type, incremental_mode) = args
     (file, timestamp) = listPath(suite, component, architecture, type,
@@ -144,7 +155,14 @@ def main():
                 elif architecture.arch_string == 'source':
                     threadpool.queueTask(writeSourceList,
                         (suite, component, Options['Incremental']))
-                elif architecture.arch_string != 'all':
+                elif architecture.arch_string == 'all':
+                    threadpool.queueTask(writeAllList,
+                        (suite, component, architecture, 'deb',
+                            Options['Incremental']))
+                    threadpool.queueTask(writeAllList,
+                        (suite, component, architecture, 'udeb',
+                            Options['Incremental']))
+                else: # arch any
                     threadpool.queueTask(writeBinaryList,
                         (suite, component, architecture, 'deb',
                             Options['Incremental']))

diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py
index a4f41f5fae0496c836c8b469260b616c4a2b1357..5ef036dad4beafbba80bb3e13d1cb988fb6e7242 100755 (executable)
--- a/dak/generate_packages_sources.py
+++ b/dak/generate_packages_sources.py
@@ -494,7 +494,7 @@ def main ():
         results=[]
         # Setup a multiprocessing Pool. As many workers as we have CPU cores.
         pool = Pool()
-        arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
+        arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session)
         Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
         for a in arch_list:
             pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]), callback=get_result)

diff --git a/daklib/lists.py b/daklib/lists.py
index a8d3cedbe03853ee5a061e200a64d25856c48935..320184e3934d19f6253bc713fe06da0e8f84cda0 100755 (executable)
--- a/daklib/lists.py
+++ b/daklib/lists.py
@@ -56,6 +56,22 @@ def getSources(suite, component, session, timestamp = None):
              'component': component.component_id }
     return fetch(query, args, session)
 
+def getArchAll(suite, component, architecture, type, session, timestamp = None):
+    '''
+    Calculates all binaries in suite and component of architecture 'all' (and
+    only 'all') and type 'deb' or 'udeb' optionally limited to binaries newer
+    than timestamp.  Returns a generator that yields a tuple of binary id and
+    full pathname to the u(deb) file. See function writeAllList() in
+    dak/generate_filelist.py for an example that uses this function.
+    '''
+    query = suite.clone(session).binaries. \
+        filter_by(architecture = architecture, binarytype = type)
+    if timestamp is not None:
+        extra_cond = 'extract(epoch from bin_associations.created) > %d' % timestamp
+        query = query.filter(extra_cond)
+    for binary in query:
+        yield (binary.binary_id, binary.poolfile.fullpath)
+
 def getBinaries(suite, component, architecture, type, session, timestamp = None):
     '''
     Calculates the binaries in suite and component of architecture and

diff --git a/daklib/queue.py b/daklib/queue.py
index 0d9ca5e1f66336dab2bdcbaca701c318a2e118a9..4ea117b30883abaa2f04a3510e4dd3cdf08af03b 100755 (executable)
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -87,6 +87,9 @@ def get_type(f, session):
         file_type = f["dbtype"]
     elif re_source_ext.match(f["type"]):
         file_type = "dsc"
+    elif f['architecture'] == 'source' and f["type"] == 'unreadable':
+        utils.warn('unreadable source file (will continue and hope for the best)')
+        return f["type"]
     else:
         file_type = f["type"]
         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))