From: Torsten Werner Date: Sun, 20 Feb 2011 13:08:29 +0000 (+0100) Subject: Implement and test class ContentsWriter. X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=commitdiff_plain;h=1b3315533c0a701972e1636e809cf30106ee5424 Implement and test class ContentsWriter. Signed-off-by: Torsten Werner --- diff --git a/daklib/contents.py b/daklib/contents.py new file mode 100755 index 00000000..7914c20e --- /dev/null +++ b/daklib/contents.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +""" +Helper code for contents generation. + +@contact: Debian FTPMaster +@copyright: 2011 Torsten Werner +@license: GNU General Public License version 2 or later +""" + +################################################################################ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +from daklib.dbconn import * +from daklib.config import Config + +from sqlalchemy import desc, or_ +from subprocess import Popen, PIPE + +class ContentsWriter(object): + ''' + ContentsWriter writes the Contents-$arch.gz files. + ''' + def __init__(self, suite, architecture, overridetype, component = None): + ''' + The constructor clones its arguments into a new session object to make + sure that the new ContentsWriter object can be executed in a different + thread. + ''' + self.suite = suite.clone() + self.session = self.suite.session() + self.architecture = architecture.clone(self.session) + self.overridetype = overridetype.clone(self.session) + if component is not None: + self.component = component.clone(self.session) + else: + self.component = None + + def query(self): + ''' + Returns a query object that is doing most of the work. + ''' + params = { + 'suite': self.suite.suite_id, + 'arch_all': get_architecture('all', self.session).arch_id, + 'arch': self.architecture.arch_id, + 'type_id': self.overridetype.overridetype_id, + 'type': self.overridetype.overridetype, + } + + if self.component is not None: + params['component'] = component.component_id + sql = ''' +create temp table newest_binaries ( + id integer primary key, + package text); + +create index newest_binaries_by_package on newest_binaries (package); + +insert into newest_binaries (id, package) + select distinct on (package) id, package from binaries + where type = :type and + (architecture = :arch_all or architecture = :arch) and + id in (select bin from bin_associations where suite = :suite) + order by package, version desc; + +with + +unique_override as + (select o.package, s.section + from override o, section s + where o.suite = :suite and o.type = :type_id and o.section = s.id and + o.component = :component) + +select bc.file, substring(o.section from position('/' in o.section) + 1) || '/' || b.package as package + from newest_binaries b, bin_contents bc, unique_override o + where b.id = bc.binary_id and o.package = b.package + order by bc.file, b.package''' + + else: + sql = ''' +create temp table newest_binaries ( + id integer primary key, + package text); + +create index newest_binaries_by_package on newest_binaries (package); + +insert into newest_binaries (id, package) + select distinct on (package) id, package from binaries + where type = :type and + (architecture = :arch_all or architecture = :arch) and + id in (select bin from bin_associations where suite = :suite) + order by package, version desc; + +with + +unique_override as + (select distinct on (o.package, s.section) o.package, s.section + from override o, section s + where o.suite = :suite and o.type = :type_id and o.section = s.id + order by o.package, s.section, o.modified desc) + +select bc.file, substring(o.section from position('/' in o.section) + 1) || '/' || b.package as package + from newest_binaries b, bin_contents bc, unique_override o + where b.id = bc.binary_id and o.package = b.package + order by bc.file, b.package''' + + return self.session.query("file", "package").from_statement(sql). \ + params(params) + + def formatline(self, filename, package_list): + ''' + Returns a formatted string for the filename argument. + ''' + package_list = ','.join(package_list) + return "%-60s%s\n" % (filename, package_list) + + def fetch(self): + ''' + Yields a new line of the Contents-$arch.gz file in filename order. + ''' + last_filename = None + package_list = [] + for filename, package in self.query().yield_per(100): + if filename != last_filename: + if last_filename is not None: + yield self.formatline(last_filename, package_list) + last_filename = filename + package_list = [] + package_list.append(package) + yield self.formatline(last_filename, package_list) + # end transaction to return connection to pool + self.session.rollback() + + def get_list(self): + ''' + Returns a list of lines for the Contents-$arch.gz file. + ''' + return [item for item in self.fetch()] + + def output_filename(self): + ''' + Returns the name of the output file. + ''' + values = { + 'root': Config()['Dir::Root'], + 'suite': self.suite.suite_name, + 'architecture': self.architecture.arch_string + } + if self.component is None: + return "%(root)s%(suite)s/Contents-%(architecture)s.gz" % values + values['component'] = self.component.component_name + return "%(root)s%(suite)s/%(component)s/Contents-%(architecture)s.gz" % values + + def write_file(self): + ''' + Write the output file. + ''' + command = ['gzip', '--rsyncable'] + output_file = open(self.output_filename(), 'w') + pipe = Popen(command, stdin = PIPE, stdout = output_file).stdin + for item in self.fetch(): + pipe.write(item) + pipe.close() + output_file.close() diff --git a/tests/db_test.py b/tests/db_test.py index 2ce786ab..4f5e6382 100644 --- a/tests/db_test.py +++ b/tests/db_test.py @@ -101,8 +101,8 @@ class DBDakTestCase(DakTestCase): if 'comp' in self.__dict__: return self.comp = {} - self.comp['main'] = Component(component_name = 'main') - self.comp['contrib'] = Component(component_name = 'contrib') + for name in ('main', 'contrib', 'non-free'): + self.comp[name] = Component(component_name = name) self.session.add_all(self.comp.values()) def setup_locations(self): @@ -322,7 +322,8 @@ class DBDakTestCase(DakTestCase): def tearDown(self): self.session.rollback() for class_ in self.classes_to_clean(): - self.session.query(class_).delete() + for object_ in self.session.query(class_): + self.session.delete(object_) self.session.commit() # usually there is no need to drop all tables here #self.metadata.drop_all() diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py index f7be2f26..9b33c04b 100755 --- a/tests/dbtest_contents.py +++ b/tests/dbtest_contents.py @@ -2,9 +2,8 @@ from db_test import DBDakTestCase -from daklib.dbconn import DBConn, BinContents, OverrideType, get_override_type, \ - Section, get_section, get_sections, Priority, get_priority, get_priorities, \ - Override, get_override +from daklib.dbconn import * +from daklib.contents import ContentsWriter from sqlalchemy.exc import FlushError, IntegrityError import unittest @@ -129,5 +128,36 @@ class ContentsTestCase(DBDakTestCase): self.assertEqual(self.override['hello_sid_main_udeb'], \ self.otype['udeb'].overrides.one()) + def test_contentswriter(self): + ''' + Test the ContentsWriter class. + ''' + self.setup_suites() + self.setup_architectures() + self.setup_overridetypes() + self.setup_binaries() + self.setup_overrides() + self.binary['hello_2.2-1_i386'].contents.append(BinContents(file = '/usr/bin/hello')) + self.session.commit() + cw = ContentsWriter(self.suite['squeeze'], self.arch['i386'], self.otype['deb']) + self.assertEqual(['/usr/bin/hello python/hello\n'], \ + cw.get_list()) + # test formatline and sort order + self.assertEqual('/usr/bin/hello python/hello\n', \ + cw.formatline('/usr/bin/hello', ['python/hello'])) + self.assertEqual('/usr/bin/hello editors/emacs,python/hello,utils/sl\n', \ + cw.formatline('/usr/bin/hello', ['editors/emacs', 'python/hello', 'utils/sl'])) + # test output_filename + self.assertEqual('tests/fixtures/ftp/squeeze/Contents-i386.gz', \ + cw.output_filename()) + cw = ContentsWriter(self.suite['squeeze'], self.arch['i386'], \ + self.otype['udeb'], self.comp['main']) + self.assertEqual('tests/fixtures/ftp/squeeze/main/Contents-i386.gz', \ + cw.output_filename()) + + def classes_to_clean(self): + return [Override, Suite, BinContents, DBBinary, DBSource, Architecture, Section, \ + OverrideType, Maintainer, Component, Priority, PoolFile] + if __name__ == '__main__': unittest.main()