From: Torsten Werner <twerner@debian.org>
Date: Tue, 22 Mar 2011 15:50:39 +0000 (+0100)
Subject: Merge branch 'contents'
X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=87bcdbc7dffd6230cb4d49942fbd87ead6f2fffe;hp=1d6c35970efaaa35a4511976510661b548e19514;p=dak.git

Merge branch 'contents'
---

diff --git a/dak/contents.py b/dak/contents.py
index 5c33afa2..d763f869 100755
--- a/dak/contents.py
+++ b/dak/contents.py
@@ -77,26 +77,11 @@ OPTIONS for scan
 
 def write_all(cnf, suite_names = [], force = None):
     Logger = daklog.Logger(cnf.Cnf, 'contents generate')
-    ContentsWriter.write_all(suite_names, force)
+    ContentsWriter.write_all(Logger, suite_names, force)
     Logger.close()
 
 ################################################################################
 
-def write_helper(suite_name, argv):
-    session = DBConn().session()
-    suite = get_suite(suite_name, session)
-    architecture = get_architecture(argv[0], session)
-    debtype = get_override_type(argv[1], session)
-    if len(argv) == 3:
-        component = get_component(argv[2], session)
-    else:
-        component = None
-    session.rollback()
-    ContentsWriter(suite, architecture, debtype, component).write_file()
-    session.close()
-
-################################################################################
-
 def scan_all(cnf, limit):
     Logger = daklog.Logger(cnf.Cnf, 'contents scan')
     result = ContentsScanner.scan_all(limit)
@@ -121,7 +106,7 @@ def main():
     args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments, sys.argv)
     options = cnf.SubTree('Contents::Options')
 
-    if (len(args) < 1) or options['Help']:
+    if (len(args) != 1) or options['Help']:
         usage()
 
     limit = None
@@ -140,10 +125,6 @@ def main():
         write_all(cnf, suite_names, force)
         return
 
-    if args[0] == 'generate_helper':
-        write_helper(suite_names[0], args[1:])
-        return
-
     usage()
 
 
diff --git a/daklib/contents.py b/daklib/contents.py
index 27ffdda6..bb815893 100755
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -32,7 +32,7 @@ from multiprocessing import Pool
 
 from sqlalchemy import desc, or_
 from sqlalchemy.exc import IntegrityError
-from subprocess import Popen, PIPE, call
+from subprocess import Popen, PIPE
 
 import os.path
 
@@ -92,10 +92,10 @@ unique_override as
         where o.suite = :overridesuite and o.type = :type_id and o.section = s.id and
         o.component = :component)
 
-select bc.file, o.section || '/' || b.package as package
+select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist
     from newest_binaries b, bin_contents bc, unique_override o
     where b.id = bc.binary_id and o.package = b.package
-    order by bc.file, b.package'''
+    group by bc.file'''
 
         else:
             sql = '''
@@ -120,36 +120,26 @@ unique_override as
         where o.suite = :overridesuite and o.type = :type_id and o.section = s.id
         order by o.package, s.section, o.modified desc)
 
-select bc.file, o.section || '/' || b.package as package
+select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist
     from newest_binaries b, bin_contents bc, unique_override o
     where b.id = bc.binary_id and o.package = b.package
-    order by bc.file, b.package'''
+    group by bc.file'''
 
-        return self.session.query("file", "package").from_statement(sql). \
+        return self.session.query("file", "pkglist").from_statement(sql). \
             params(params)
 
     def formatline(self, filename, package_list):
         '''
         Returns a formatted string for the filename argument.
         '''
-        package_list = ','.join(package_list)
         return "%-55s %s\n" % (filename, package_list)
 
     def fetch(self):
         '''
         Yields a new line of the Contents-$arch.gz file in filename order.
         '''
-        last_filename = None
-        package_list = []
-        for filename, package in self.query().yield_per(100):
-            if filename != last_filename:
-                if last_filename is not None:
-                    yield self.formatline(last_filename, package_list)
-                last_filename = filename
-                package_list = []
-            package_list.append(package)
-        if last_filename is not None:
-            yield self.formatline(last_filename, package_list)
+        for filename, package_list in self.query().yield_per(100):
+            yield self.formatline(filename, package_list)
         # end transaction to return connection to pool
         self.session.rollback()
 
@@ -201,41 +191,74 @@ select bc.file, o.section || '/' || b.package as package
         gzip.stdin.close()
         output_file.close()
         gzip.wait()
-        os.remove(final_filename)
+        try:
+            os.remove(final_filename)
+        except:
+            pass
         os.rename(temp_filename, final_filename)
         os.chmod(final_filename, 0664)
 
     @classmethod
-    def write_all(class_, suite_names = [], force = False):
+    def log_result(class_, result):
+        '''
+        Writes a result message to the logfile.
+        '''
+        class_.logger.log(result)
+
+    @classmethod
+    def write_all(class_, logger, suite_names = [], force = False):
         '''
         Writes all Contents files for suites in list suite_names which defaults
         to all 'touchable' suites if not specified explicitely. Untouchable
         suites will be included if the force argument is set to True.
         '''
+        class_.logger = logger
         session = DBConn().session()
         suite_query = session.query(Suite)
         if len(suite_names) > 0:
             suite_query = suite_query.filter(Suite.suite_name.in_(suite_names))
         if not force:
             suite_query = suite_query.filter_by(untouchable = False)
+        deb_id = get_override_type('deb', session).overridetype_id
+        udeb_id = get_override_type('udeb', session).overridetype_id
+        main_id = get_component('main', session).component_id
+        non_free_id = get_component('non-free', session).component_id
         pool = Pool()
         for suite in suite_query:
+            suite_id = suite.suite_id
             for architecture in suite.get_architectures(skipsrc = True, skipall = True):
+                arch_id = architecture.arch_id
                 # handle 'deb' packages
-                command = ['dak', 'contents', '-s', suite.suite_name, \
-                    'generate_helper', architecture.arch_string, 'deb']
-                pool.apply_async(call, (command, ))
+                pool.apply_async(generate_helper, (suite_id, arch_id, deb_id), \
+                    callback = class_.log_result)
                 # handle 'udeb' packages for 'main' and 'non-free'
-                command = ['dak', 'contents', '-s', suite.suite_name, \
-                    'generate_helper', architecture.arch_string, 'udeb', 'main']
-                pool.apply_async(call, (command, ))
-                command = ['dak', 'contents', '-s', suite.suite_name, \
-                    'generate_helper', architecture.arch_string, 'udeb', 'non-free']
-                pool.apply_async(call, (command, ))
+                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, main_id), \
+                    callback = class_.log_result)
+                pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, non_free_id), \
+                    callback = class_.log_result)
         pool.close()
         pool.join()
         session.close()
 
+def generate_helper(suite_id, arch_id, overridetype_id, component_id = None):
+    '''
+    This function is called in a new subprocess.
+    '''
+    DBConn().reset()
+    session = DBConn().session()
+    suite = Suite.get(suite_id, session)
+    architecture = Architecture.get(arch_id, session)
+    overridetype = OverrideType.get(overridetype_id, session)
+    log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype]
+    if component_id is None:
+        component = None
+    else:
+        component = Component.get(component_id, session)
+        log_message.append(component.component_name)
+    contents_writer = ContentsWriter(suite, architecture, overridetype, component)
+    contents_writer.write_file()
+    return log_message
+
 
 class ContentsScanner(object):
     '''
diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py
index 43cf838c..622e4aaf 100755
--- a/tests/dbtest_contents.py
+++ b/tests/dbtest_contents.py
@@ -145,9 +145,7 @@ class ContentsTestCase(DBDakTestCase):
             cw.get_list())
         # test formatline and sort order
         self.assertEqual('/usr/bin/hello                                          python/hello\n', \
-            cw.formatline('/usr/bin/hello', ['python/hello']))
-        self.assertEqual('/usr/bin/hello                                          editors/emacs,python/hello,utils/sl\n', \
-            cw.formatline('/usr/bin/hello', ['editors/emacs', 'python/hello', 'utils/sl']))
+            cw.formatline('/usr/bin/hello', 'python/hello'))
         # test output_filename
         self.assertEqual('tests/fixtures/ftp/dists/squeeze/Contents-i386.gz', \
             normpath(cw.output_filename()))
diff --git a/tests/fixtures/dak.conf b/tests/fixtures/dak.conf
index abfa2511..6f68238d 100644
--- a/tests/fixtures/dak.conf
+++ b/tests/fixtures/dak.conf
@@ -29,5 +29,6 @@ DB
   Name "test_projectb";
   Host "";
   Port -1;
+  Unicode false;
 };