from daklib.dbconn import *
from daklib.config import Config
+from daklib.filewriter import BinaryContentsFileWriter, SourceContentsFileWriter
from multiprocessing import Pool
from shutil import rmtree
from tempfile import mkdtemp
import os.path
+import signal
class BinaryContentsWriter(object):
'''
'''
return [item for item in self.fetch()]
- def output_filename(self):
+ def writer(self):
'''
- Returns the name of the output file.
+ Returns a writer object.
'''
values = {
- 'root': Config()['Dir::Root'],
- 'suite': self.suite.suite_name,
- 'architecture': self.architecture.arch_string
+ 'suite': self.suite.suite_name,
+ 'architecture': self.architecture.arch_string,
}
- if self.component is None:
- return "%(root)s/dists/%(suite)s/Contents-%(architecture)s.gz" % values
- values['component'] = self.component.component_name
- return "%(root)s/dists/%(suite)s/%(component)s/Contents-%(architecture)s.gz" % values
+ if self.component is not None:
+ values['component'] = self.component.component_name
+ return BinaryContentsFileWriter(**values)
def get_header(self):
'''
'''
Write the output file.
'''
- command = ['gzip', '--rsyncable']
- final_filename = self.output_filename()
- temp_filename = final_filename + '.new'
- output_file = open(temp_filename, 'w')
- gzip = Popen(command, stdin = PIPE, stdout = output_file)
- gzip.stdin.write(self.get_header())
+ writer = self.writer()
+ file = writer.open()
+ file.write(self.get_header())
for item in self.fetch():
- gzip.stdin.write(item)
- gzip.stdin.close()
- output_file.close()
- gzip.wait()
- os.chmod(temp_filename, 0664)
- os.rename(temp_filename, final_filename)
+ file.write(item)
+ writer.close()
class SourceContentsWriter(object):
'''
return [item for item in self.fetch()]
- def output_filename(self):
+ def writer(self):
'''
- Returns the name of the output file.
+ Returns a writer object.
'''
values = {
- 'root': Config()['Dir::Root'],
'suite': self.suite.suite_name,
'component': self.component.component_name
}
- return "%(root)s/dists/%(suite)s/%(component)s/Contents-source.gz" % values
+ return SourceContentsFileWriter(**values)
def write_file(self):
'''
Write the output file.
'''
- command = ['gzip', '--rsyncable']
- final_filename = self.output_filename()
- temp_filename = final_filename + '.new'
- output_file = open(temp_filename, 'w')
- gzip = Popen(command, stdin = PIPE, stdout = output_file)
+ writer = self.writer()
+ file = writer.open()
for item in self.fetch():
- gzip.stdin.write(item)
- gzip.stdin.close()
- output_file.close()
- gzip.wait()
- os.chmod(temp_filename, 0664)
- os.rename(temp_filename, final_filename)
+ file.write(item)
+ writer.close()
-def generate_helper(suite_id, arch_id, overridetype_id, component_id = None):
+def binary_helper(suite_id, arch_id, overridetype_id, component_id = None):
'''
- This function is called in a new subprocess.
+ This function is called in a new subprocess and multiprocessing wants a top
+ level function.
'''
- session = DBConn().session()
+ session = DBConn().session(work_mem = 1000)
suite = Suite.get(suite_id, session)
architecture = Architecture.get(arch_id, session)
overridetype = OverrideType.get(overridetype_id, session)
contents_writer.write_file()
return log_message
+def source_helper(suite_id, component_id):
+ '''
+ This function is called in a new subprocess and multiprocessing wants a top
+ level function.
+ '''
+ session = DBConn().session(work_mem = 1000)
+ suite = Suite.get(suite_id, session)
+ component = Component.get(component_id, session)
+ log_message = [suite.suite_name, 'source', component.component_name]
+ contents_writer = SourceContentsWriter(suite, component)
+ contents_writer.write_file()
+ return log_message
+
class ContentsWriter(object):
'''
Loop over all suites, architectures, overridetypes, and components to write
deb_id = get_override_type('deb', session).overridetype_id
udeb_id = get_override_type('udeb', session).overridetype_id
main_id = get_component('main', session).component_id
+ contrib_id = get_component('contrib', session).component_id
non_free_id = get_component('non-free', session).component_id
pool = Pool()
for suite in suite_query:
suite_id = suite.suite_id
+ # handle source packages
+ pool.apply_async(source_helper, (suite_id, main_id),
+ callback = class_.log_result)
+ pool.apply_async(source_helper, (suite_id, contrib_id),
+ callback = class_.log_result)
+ pool.apply_async(source_helper, (suite_id, non_free_id),
+ callback = class_.log_result)
for architecture in suite.get_architectures(skipsrc = True, skipall = True):
arch_id = architecture.arch_id
# handle 'deb' packages
- pool.apply_async(generate_helper, (suite_id, arch_id, deb_id), \
+ pool.apply_async(binary_helper, (suite_id, arch_id, deb_id), \
callback = class_.log_result)
# handle 'udeb' packages for 'main' and 'non-free'
- pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, main_id), \
+ pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, main_id), \
callback = class_.log_result)
- pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, non_free_id), \
+ pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, non_free_id), \
callback = class_.log_result)
pool.close()
pool.join()
scanner.scan()
+def subprocess_setup():
+ # Python installs a SIGPIPE handler by default. This is usually not what
+ # non-Python subprocesses expect.
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
class UnpackedSource(object):
'''
UnpackedSource extracts a source package into a temporary location and
'''
The dscfilename is a name of a DSC file that will be extracted.
'''
- self.root_directory = os.path.join(mkdtemp(), 'root')
- command = ('dpkg-source', '--no-copy', '--no-check', '-x', dscfilename,
- self.root_directory)
- # dpkg-source does not have a --quiet option
- devnull = open(os.devnull, 'w')
- check_call(command, stdout = devnull, stderr = devnull)
- devnull.close()
+ temp_directory = mkdtemp(dir = Config()['Dir::TempPath'])
+ self.root_directory = os.path.join(temp_directory, 'root')
+ command = ('dpkg-source', '--no-copy', '--no-check', '-q', '-x',
+ dscfilename, self.root_directory)
+ check_call(command, preexec_fn = subprocess_setup)
def get_root_directory(self):
'''
projects / dak.git / blobdiff