Merge remote branch 'ftpmaster/master' into psimport

[dak.git] / daklib / contents.py

diff --git a/daklib/contents.py b/daklib/contents.py
index 848976d54717100992daf4099ac1f4a947dd2a98..a158e8fcd4bb7d2ed27e8547b79b03d1cbd84b8a 100755 (executable)
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -27,12 +27,11 @@ Helper code for contents generation.
 
 from daklib.dbconn import *
 from daklib.config import Config
 
 from daklib.dbconn import *
 from daklib.config import Config

-from daklib.threadpool import ThreadPool
-from multiprocessing import Pool

 
 

-from sqlalchemy import desc, or_
-from sqlalchemy.exc import IntegrityError
-from subprocess import Popen, PIPE
+from multiprocessing import Pool
+from shutil import rmtree
+from subprocess import Popen, PIPE, check_call
+from tempfile import mkdtemp

 
 import os.path
 
 
 import os.path
 


@@ -199,12 +198,20 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package
         os.chmod(final_filename, 0664)
 
     @classmethod
         os.chmod(final_filename, 0664)
 
     @classmethod

-    def write_all(class_, suite_names = [], force = False):
+    def log_result(class_, result):
+        '''
+        Writes a result message to the logfile.
+        '''
+        class_.logger.log(result)
+
+    @classmethod
+    def write_all(class_, logger, suite_names = [], force = False):

         '''
         Writes all Contents files for suites in list suite_names which defaults
         to all 'touchable' suites if not specified explicitely. Untouchable
         suites will be included if the force argument is set to True.
         '''
         '''
         Writes all Contents files for suites in list suite_names which defaults
         to all 'touchable' suites if not specified explicitely. Untouchable
         suites will be included if the force argument is set to True.
         '''

+        class_.logger = logger

         session = DBConn().session()
         suite_query = session.query(Suite)
         if len(suite_names) > 0:
         session = DBConn().session()
         suite_query = session.query(Suite)
         if len(suite_names) > 0:


@@ -221,10 +228,13 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package
             for architecture in suite.get_architectures(skipsrc = True, skipall = True):
                 arch_id = architecture.arch_id
                 # handle 'deb' packages
             for architecture in suite.get_architectures(skipsrc = True, skipall = True):
                 arch_id = architecture.arch_id
                 # handle 'deb' packages

-                pool.apply_async(generate_helper, (suite_id, arch_id, deb_id))
+                pool.apply_async(generate_helper, (suite_id, arch_id, deb_id), \
+                    callback = class_.log_result)

                 # handle 'udeb' packages for 'main' and 'non-free'
                 # handle 'udeb' packages for 'main' and 'non-free'

-                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, main_id))
-                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, non_free_id))
+                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, main_id), \
+                    callback = class_.log_result)
+                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, non_free_id), \
+                    callback = class_.log_result)

         pool.close()
         pool.join()
         session.close()
         pool.close()
         pool.join()
         session.close()


@@ -233,17 +243,19 @@ def generate_helper(suite_id, arch_id, overridetype_id, component_id = None):
     '''
     This function is called in a new subprocess.
     '''
     '''
     This function is called in a new subprocess.
     '''

-    DBConn().reset()

     session = DBConn().session()
     suite = Suite.get(suite_id, session)
     architecture = Architecture.get(arch_id, session)
     overridetype = OverrideType.get(overridetype_id, session)
     session = DBConn().session()
     suite = Suite.get(suite_id, session)
     architecture = Architecture.get(arch_id, session)
     overridetype = OverrideType.get(overridetype_id, session)

+    log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype]

     if component_id is None:
         component = None
     else:
         component = Component.get(component_id, session)
     if component_id is None:
         component = None
     else:
         component = Component.get(component_id, session)

+        log_message.append(component.component_name)

     contents_writer = ContentsWriter(suite, architecture, overridetype, component)
     contents_writer.write_file()
     contents_writer = ContentsWriter(suite, architecture, overridetype, component)
     contents_writer.write_file()

+    return log_message

 
 
 class ContentsScanner(object):
 
 
 class ContentsScanner(object):


@@ -251,12 +263,12 @@ class ContentsScanner(object):
     ContentsScanner provides a threadsafe method scan() to scan the contents of
     a DBBinary object.
     '''
     ContentsScanner provides a threadsafe method scan() to scan the contents of
     a DBBinary object.
     '''

-    def __init__(self, binary):
+    def __init__(self, binary_id):

         '''
         '''

-        The argument binary is the actual DBBinary object that should be
-        scanned.
+        The argument binary_id is the id of the DBBinary object that
+        should be scanned.

         '''
         '''

-        self.binary_id = binary.binary_id
+        self.binary_id = binary_id

 
     def scan(self, dummy_arg = None):
         '''
 
     def scan(self, dummy_arg = None):
         '''


@@ -288,10 +300,79 @@ class ContentsScanner(object):
         if limit is not None:
             query = query.limit(limit)
         processed = query.count()
         if limit is not None:
             query = query.limit(limit)
         processed = query.count()

-        threadpool = ThreadPool()
+        pool = Pool()

         for binary in query.yield_per(100):
         for binary in query.yield_per(100):

-            threadpool.queueTask(ContentsScanner(binary).scan)
-        threadpool.joinAll()
+            pool.apply_async(scan_helper, (binary.binary_id, ))
+        pool.close()
+        pool.join()

         remaining = remaining()
         session.close()
         return { 'processed': processed, 'remaining': remaining }
         remaining = remaining()
         session.close()
         return { 'processed': processed, 'remaining': remaining }

+
+def scan_helper(binary_id):
+    '''
+    This function runs in a subprocess.
+    '''
+    scanner = ContentsScanner(binary_id)
+    scanner.scan()
+
+
+class UnpackedSource(object):
+    '''
+    UnpackedSource extracts a source package into a temporary location and
+    gives you some convinient function for accessing it.
+    '''
+    def __init__(self, dscfilename):
+        '''
+        The dscfilename is a name of a DSC file that will be extracted.
+        '''
+        self.root_directory = os.path.join(mkdtemp(), 'root')
+        command = ('dpkg-source', '--no-copy', '--no-check', '-x', dscfilename,
+            self.root_directory)
+        # dpkg-source does not have a --quiet option
+        devnull = open(os.devnull, 'w')
+        check_call(command, stdout = devnull, stderr = devnull)
+        devnull.close()
+
+    def get_root_directory(self):
+        '''
+        Returns the name of the package's root directory which is the directory
+        where the debian subdirectory is located.
+        '''
+        return self.root_directory
+
+    def get_changelog_file(self):
+        '''
+        Returns a file object for debian/changelog or None if no such file exists.
+        '''
+        changelog_name = os.path.join(self.root_directory, 'debian', 'changelog')
+        try:
+            return open(changelog_name)
+        except IOError:
+            return None
+
+    def get_all_filenames(self):
+        '''
+        Returns an iterator over all filenames. The filenames will be relative
+        to the root directory.
+        '''
+        skip = len(self.root_directory) + 1
+        for root, _, files in os.walk(self.root_directory):
+            for name in files:
+                yield os.path.join(root[skip:], name)
+
+    def cleanup(self):
+        '''
+        Removes all temporary files.
+        '''
+        if self.root_directory is None:
+            return
+        parent_directory = os.path.dirname(self.root_directory)
+        rmtree(parent_directory)
+        self.root_directory = None
+
+    def __del__(self):
+        '''
+        Enforce cleanup.
+        '''
+        self.cleanup()