From: Mark Hymers Date: Sat, 26 Mar 2011 11:15:16 +0000 (+0000) Subject: Merge branch 'master' of ssh://ftp-master.debian.org/srv/ftp.debian.org/git/dak X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=2c2313c5af68d2bda43659c9d48b2963ef926955;hp=3cff7f4ee092a25c35040e11bae8f4c5df2d53fa;p=dak.git Merge branch 'master' of ssh://ftp-master.debian.org/srv/ftp.debian.org/git/dak --- diff --git a/daklib/contents.py b/daklib/contents.py index 449fb88e..a906b54d 100755 --- a/daklib/contents.py +++ b/daklib/contents.py @@ -27,6 +27,7 @@ Helper code for contents generation. from daklib.dbconn import * from daklib.config import Config +from daklib.filewriter import BinaryContentsFileWriter, SourceContentsFileWriter from multiprocessing import Pool from shutil import rmtree @@ -144,19 +145,17 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package ''' return [item for item in self.fetch()] - def output_filename(self): + def writer(self): ''' - Returns the name of the output file. + Returns a writer object. ''' values = { - 'root': Config()['Dir::Root'], - 'suite': self.suite.suite_name, - 'architecture': self.architecture.arch_string + 'suite': self.suite.suite_name, + 'architecture': self.architecture.arch_string, } - if self.component is None: - return "%(root)s/dists/%(suite)s/Contents-%(architecture)s.gz" % values - values['component'] = self.component.component_name - return "%(root)s/dists/%(suite)s/%(component)s/Contents-%(architecture)s.gz" % values + if self.component is not None: + values['component'] = self.component.component_name + return BinaryContentsFileWriter(**values) def get_header(self): ''' @@ -175,19 +174,12 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package ''' Write the output file. ''' - command = ['gzip', '--rsyncable'] - final_filename = self.output_filename() - temp_filename = final_filename + '.new' - output_file = open(temp_filename, 'w') - gzip = Popen(command, stdin = PIPE, stdout = output_file) - gzip.stdin.write(self.get_header()) + writer = self.writer() + file = writer.open() + file.write(self.get_header()) for item in self.fetch(): - gzip.stdin.write(item) - gzip.stdin.close() - output_file.close() - gzip.wait() - os.chmod(temp_filename, 0664) - os.rename(temp_filename, final_filename) + file.write(item) + writer.close() class SourceContentsWriter(object): @@ -251,33 +243,25 @@ select sc.file, string_agg(s.source, ',' order by s.source) as pkglist ''' return [item for item in self.fetch()] - def output_filename(self): + def writer(self): ''' - Returns the name of the output file. + Returns a writer object. ''' values = { - 'root': Config()['Dir::Root'], 'suite': self.suite.suite_name, 'component': self.component.component_name } - return "%(root)s/dists/%(suite)s/%(component)s/Contents-source.gz" % values + return SourceContentsFileWriter(**values) def write_file(self): ''' Write the output file. ''' - command = ['gzip', '--rsyncable'] - final_filename = self.output_filename() - temp_filename = final_filename + '.new' - output_file = open(temp_filename, 'w') - gzip = Popen(command, stdin = PIPE, stdout = output_file) + writer = self.writer() + file = writer.open() for item in self.fetch(): - gzip.stdin.write(item) - gzip.stdin.close() - output_file.close() - gzip.wait() - os.chmod(temp_filename, 0664) - os.rename(temp_filename, final_filename) + file.write(item) + writer.close() def binary_helper(suite_id, arch_id, overridetype_id, component_id = None): diff --git a/daklib/filewriter.py b/daklib/filewriter.py new file mode 100755 index 00000000..714531af --- /dev/null +++ b/daklib/filewriter.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +""" +Helper code for file writing with optional compression. + +@contact: Debian FTPMaster +@copyright: 2011 Torsten Werner +@license: GNU General Public License version 2 or later +""" + +################################################################################ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +from daklib.config import Config + +from subprocess import check_call + +import os, os.path + +class BaseFileWriter(object): + ''' + Base class for compressed and uncompressed file writing. + ''' + def __init__(self, template, **keywords): + ''' + The template argument is a string template like + "dists/%(suite)s/%(component)s/Contents-%(architecture)s.gz" that + should be relative to the archive's root directory. The keywords + include strings for suite, component, architecture and booleans + uncompressed, gzip, bzip2. + ''' + self.uncompressed = keywords.get('uncompressed', True) + self.gzip = keywords.get('gzip', False) + self.bzip2 = keywords.get('bzip2', False) + root_dir = Config()['Dir::Root'] + relative_dir = template % keywords + self.path = os.path.join(root_dir, relative_dir) + + def open(self): + ''' + Returns a file object for writing. + ''' + self.file = open(self.path + '.new', 'w') + return self.file + + # internal helper function + def rename(self, filename): + tempfilename = filename + '.new' + os.chmod(tempfilename, 0664) + os.rename(tempfilename, filename) + + def close(self): + ''' + Closes the file object and does the compression and rename work. + ''' + self.file.close() + if self.gzip: + check_call('gzip --rsyncable <%s.new >%s.gz.new' % (self.path, self.path), + shell = True) + self.rename('%s.gz' % self.path) + if self.bzip2: + check_call('bzip2 <%s.new >%s.bz2.new' % (self.path, self.path), shell = True) + self.rename('%s.bz2' % self.path) + if self.uncompressed: + self.rename(self.path) + else: + os.unlink(self.path + '.new') + +class BinaryContentsFileWriter(BaseFileWriter): + def __init__(self, **keywords): + ''' + The value of the keywords suite, component, and architecture are + strings. The value of component may be omitted if not applicable. + Output files are gzip compressed only. + ''' + flags = { + 'uncompressed': False, + 'gzip': True, + 'bzip2': False + } + flags.update(keywords) + if 'component' in flags: + template = "dists/%(suite)s/%(component)s/Contents-%(architecture)s" + else: + template = "dists/%(suite)s/Contents-%(architecture)s" + BaseFileWriter.__init__(self, template, **flags) + +class SourceContentsFileWriter(BaseFileWriter): + def __init__(self, **keywords): + ''' + The value of the keywords suite and component are strings. + Output files are gzip compressed only. + ''' + flags = { + 'uncompressed': False, + 'gzip': True, + 'bzip2': False + } + flags.update(keywords) + template = "dists/%(suite)s/%(component)s/Contents-source" + BaseFileWriter.__init__(self, template, **flags) + +class PackagesFileWriter(BaseFileWriter): + def __init__(self, **keywords): + ''' + The value of the keywords suite, component, debtype and architecture + are strings. Output files are gzip compressed only. + ''' + flags = { + 'uncompressed': False, + 'gzip': True, + 'bzip2': True + } + flags.update(keywords) + if flags['debtype'] == 'deb': + template = "dists/%(suite)s/%(component)s/binary-%(architecture)s/Packages" + else: # udeb + template = "dists/%(suite)s/%(component)s/debian-installer/binary-%(architecture)s/Packages" + BaseFileWriter.__init__(self, template, **flags) + +class SourcesFileWriter(BaseFileWriter): + def __init__(self, **keywords): + ''' + The value of the keywords suite and component are strings. Output + files are gzip compressed only. + ''' + flags = { + 'uncompressed': False, + 'gzip': True, + 'bzip2': True + } + flags.update(keywords) + template = "dists/%(suite)s/%(component)s/source/Sources" + BaseFileWriter.__init__(self, template, **flags)