X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fcontents.py;h=a906b54d20e2cb8f75a83c525449a73bd601be68;hb=03752f74fb17550b34678380321e2f37e64acb65;hp=7e9aaf2e1689c0ece950db4d420ab1d22ce80dcb;hpb=679f792474b393494a0560f029952079281f1ef0;p=dak.git

diff --git a/daklib/contents.py b/daklib/contents.py
index 7e9aaf2e..a906b54d 100755
--- a/daklib/contents.py
+++ b/daklib/contents.py
@@ -27,6 +27,7 @@ Helper code for contents generation.
 
 from daklib.dbconn import *
 from daklib.config import Config
+from daklib.filewriter import BinaryContentsFileWriter, SourceContentsFileWriter
 
 from multiprocessing import Pool
 from shutil import rmtree
@@ -34,6 +35,7 @@ from subprocess import Popen, PIPE, check_call
 from tempfile import mkdtemp
 
 import os.path
+import signal
 
 class BinaryContentsWriter(object):
     '''
@@ -143,19 +145,17 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package
         '''
         return [item for item in self.fetch()]
 
-    def output_filename(self):
+    def writer(self):
         '''
-        Returns the name of the output file.
+        Returns a writer object.
         '''
         values = {
-            'root': Config()['Dir::Root'],
-            'suite': self.suite.suite_name,
-            'architecture': self.architecture.arch_string
+            'suite':        self.suite.suite_name,
+            'architecture': self.architecture.arch_string,
         }
-        if self.component is None:
-            return "%(root)s/dists/%(suite)s/Contents-%(architecture)s.gz" % values
-        values['component'] = self.component.component_name
-        return "%(root)s/dists/%(suite)s/%(component)s/Contents-%(architecture)s.gz" % values
+        if self.component is not None:
+            values['component'] = self.component.component_name
+        return BinaryContentsFileWriter(**values)
 
     def get_header(self):
         '''
@@ -174,19 +174,12 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package
         '''
         Write the output file.
         '''
-        command = ['gzip', '--rsyncable']
-        final_filename = self.output_filename()
-        temp_filename = final_filename + '.new'
-        output_file = open(temp_filename, 'w')
-        gzip = Popen(command, stdin = PIPE, stdout = output_file)
-        gzip.stdin.write(self.get_header())
+        writer = self.writer()
+        file = writer.open()
+        file.write(self.get_header())
         for item in self.fetch():
-            gzip.stdin.write(item)
-        gzip.stdin.close()
-        output_file.close()
-        gzip.wait()
-        os.chmod(temp_filename, 0664)
-        os.rename(temp_filename, final_filename)
+            file.write(item)
+        writer.close()
 
 
 class SourceContentsWriter(object):
@@ -250,33 +243,25 @@ select sc.file, string_agg(s.source, ',' order by s.source) as pkglist
         '''
         return [item for item in self.fetch()]
 
-    def output_filename(self):
+    def writer(self):
         '''
-        Returns the name of the output file.
+        Returns a writer object.
         '''
         values = {
-            'root':      Config()['Dir::Root'],
             'suite':     self.suite.suite_name,
             'component': self.component.component_name
         }
-        return "%(root)s/dists/%(suite)s/%(component)s/Contents-source.gz" % values
+        return SourceContentsFileWriter(**values)
 
     def write_file(self):
         '''
         Write the output file.
         '''
-        command = ['gzip', '--rsyncable']
-        final_filename = self.output_filename()
-        temp_filename = final_filename + '.new'
-        output_file = open(temp_filename, 'w')
-        gzip = Popen(command, stdin = PIPE, stdout = output_file)
+        writer = self.writer()
+        file = writer.open()
         for item in self.fetch():
-            gzip.stdin.write(item)
-        gzip.stdin.close()
-        output_file.close()
-        gzip.wait()
-        os.chmod(temp_filename, 0664)
-        os.rename(temp_filename, final_filename)
+            file.write(item)
+        writer.close()
 
 
 def binary_helper(suite_id, arch_id, overridetype_id, component_id = None):
@@ -284,7 +269,7 @@ def binary_helper(suite_id, arch_id, overridetype_id, component_id = None):
     This function is called in a new subprocess and multiprocessing wants a top
     level function.
     '''
-    session = DBConn().session()
+    session = DBConn().session(work_mem = 1000)
     suite = Suite.get(suite_id, session)
     architecture = Architecture.get(arch_id, session)
     overridetype = OverrideType.get(overridetype_id, session)
@@ -303,10 +288,10 @@ def source_helper(suite_id, component_id):
     This function is called in a new subprocess and multiprocessing wants a top
     level function.
     '''
-    session = DBConn().session()
+    session = DBConn().session(work_mem = 1000)
     suite = Suite.get(suite_id, session)
     component = Component.get(component_id, session)
-    log_message = [suite.suite_name, component.component_name]
+    log_message = [suite.suite_name, 'source', component.component_name]
     contents_writer = SourceContentsWriter(suite, component)
     contents_writer.write_file()
     return log_message
@@ -426,6 +411,11 @@ def binary_scan_helper(binary_id):
     scanner.scan()
 
 
+def subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
 class UnpackedSource(object):
     '''
     UnpackedSource extracts a source package into a temporary location and
@@ -435,13 +425,11 @@ class UnpackedSource(object):
         '''
         The dscfilename is a name of a DSC file that will be extracted.
         '''
-        self.root_directory = os.path.join(mkdtemp(), 'root')
-        command = ('dpkg-source', '--no-copy', '--no-check', '-x', dscfilename,
-            self.root_directory)
-        # dpkg-source does not have a --quiet option
-        devnull = open(os.devnull, 'w')
-        check_call(command, stdout = devnull, stderr = devnull)
-        devnull.close()
+        temp_directory = mkdtemp(dir = Config()['Dir::TempPath'])
+        self.root_directory = os.path.join(temp_directory, 'root')
+        command = ('dpkg-source', '--no-copy', '--no-check', '-q', '-x',
+            dscfilename, self.root_directory)
+        check_call(command, preexec_fn = subprocess_setup)
 
     def get_root_directory(self):
         '''