From: Torsten Werner Date: Thu, 24 Mar 2011 13:26:26 +0000 (+0100) Subject: Merge branch 'master' into contents X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=42658acd4f03a531537afb8748c0459be3a7c386;hp=bd340bdc82bbed64405cdcf0beb21fa8cf16f218;p=dak.git Merge branch 'master' into contents --- diff --git a/daklib/contents.py b/daklib/contents.py index 2a29b2e5..0fa52df2 100755 --- a/daklib/contents.py +++ b/daklib/contents.py @@ -35,16 +35,11 @@ from tempfile import mkdtemp import os.path -class ContentsWriter(object): +class BinaryContentsWriter(object): ''' - ContentsWriter writes the Contents-$arch.gz files. + BinaryContentsWriter writes the Contents-$arch.gz files. ''' def __init__(self, suite, architecture, overridetype, component = None): - ''' - The constructor clones its arguments into a new session object to make - sure that the new ContentsWriter object can be executed in a different - thread. - ''' self.suite = suite self.architecture = architecture self.overridetype = overridetype @@ -193,6 +188,134 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package os.chmod(temp_filename, 0664) os.rename(temp_filename, final_filename) + +class SourceContentsWriter(object): + ''' + SourceContentsWriter writes the Contents-source.gz files. + ''' + def __init__(self, suite, component): + self.suite = suite + self.component = component + self.session = suite.session() + + def query(self): + ''' + Returns a query object that is doing most of the work. + ''' + params = { + 'suite_id': self.suite.suite_id, + 'component_id': self.component.component_id, + } + + sql = ''' +create temp table newest_sources ( + id integer primary key, + source text); + +create index sources_binaries_by_source on newest_sources (source); + +insert into newest_sources (id, source) + select distinct on (source) s.id, s.source from source s + join files f on f.id = s.file + join location l on l.id = f.location + where s.id in (select source from src_associations where suite = :suite_id) + and l.component = :component_id + order by source, version desc; + +select sc.file, string_agg(s.source, ',' order by s.source) as pkglist + from newest_sources s, src_contents sc + where s.id = sc.source_id group by sc.file''' + + return self.session.query("file", "pkglist").from_statement(sql). \ + params(params) + + def formatline(self, filename, package_list): + ''' + Returns a formatted string for the filename argument. + ''' + return "%s\t%s\n" % (filename, package_list) + + def fetch(self): + ''' + Yields a new line of the Contents-source.gz file in filename order. + ''' + for filename, package_list in self.query().yield_per(100): + yield self.formatline(filename, package_list) + # end transaction to return connection to pool + self.session.rollback() + + def get_list(self): + ''' + Returns a list of lines for the Contents-source.gz file. + ''' + return [item for item in self.fetch()] + + def output_filename(self): + ''' + Returns the name of the output file. + ''' + values = { + 'root': Config()['Dir::Root'], + 'suite': self.suite.suite_name, + 'component': self.component.component_name + } + return "%(root)s/dists/%(suite)s/%(component)s/Contents-source.gz" % values + + def write_file(self): + ''' + Write the output file. + ''' + command = ['gzip', '--rsyncable'] + final_filename = self.output_filename() + temp_filename = final_filename + '.new' + output_file = open(temp_filename, 'w') + gzip = Popen(command, stdin = PIPE, stdout = output_file) + for item in self.fetch(): + gzip.stdin.write(item) + gzip.stdin.close() + output_file.close() + gzip.wait() + os.chmod(temp_filename, 0664) + os.rename(temp_filename, final_filename) + + +def binary_helper(suite_id, arch_id, overridetype_id, component_id = None): + ''' + This function is called in a new subprocess and multiprocessing wants a top + level function. + ''' + session = DBConn().session() + suite = Suite.get(suite_id, session) + architecture = Architecture.get(arch_id, session) + overridetype = OverrideType.get(overridetype_id, session) + log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype] + if component_id is None: + component = None + else: + component = Component.get(component_id, session) + log_message.append(component.component_name) + contents_writer = BinaryContentsWriter(suite, architecture, overridetype, component) + contents_writer.write_file() + return log_message + +def source_helper(suite_id, component_id): + ''' + This function is called in a new subprocess and multiprocessing wants a top + level function. + ''' + session = DBConn().session() + suite = Suite.get(suite_id, session) + component = Component.get(component_id, session) + log_message = [suite.suite_name, 'source', component.component_name] + contents_writer = SourceContentsWriter(suite, component) + contents_writer.write_file() + return log_message + +class ContentsWriter(object): + ''' + Loop over all suites, architectures, overridetypes, and components to write + all contents files. + ''' @classmethod def log_result(class_, result): ''' @@ -217,42 +340,32 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package deb_id = get_override_type('deb', session).overridetype_id udeb_id = get_override_type('udeb', session).overridetype_id main_id = get_component('main', session).component_id + contrib_id = get_component('contrib', session).component_id non_free_id = get_component('non-free', session).component_id pool = Pool() for suite in suite_query: suite_id = suite.suite_id + # handle source packages + pool.apply_async(source_helper, (suite_id, main_id), + callback = class_.log_result) + pool.apply_async(source_helper, (suite_id, contrib_id), + callback = class_.log_result) + pool.apply_async(source_helper, (suite_id, non_free_id), + callback = class_.log_result) for architecture in suite.get_architectures(skipsrc = True, skipall = True): arch_id = architecture.arch_id # handle 'deb' packages - pool.apply_async(generate_helper, (suite_id, arch_id, deb_id), \ + pool.apply_async(binary_helper, (suite_id, arch_id, deb_id), \ callback = class_.log_result) # handle 'udeb' packages for 'main' and 'non-free' - pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, main_id), \ + pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, main_id), \ callback = class_.log_result) - pool.apply_async(generate_helper, (suite_id, arch_id, udeb_id, non_free_id), \ + pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, non_free_id), \ callback = class_.log_result) pool.close() pool.join() session.close() -def generate_helper(suite_id, arch_id, overridetype_id, component_id = None): - ''' - This function is called in a new subprocess. - ''' - session = DBConn().session() - suite = Suite.get(suite_id, session) - architecture = Architecture.get(arch_id, session) - overridetype = OverrideType.get(overridetype_id, session) - log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype] - if component_id is None: - component = None - else: - component = Component.get(component_id, session) - log_message.append(component.component_name) - contents_writer = ContentsWriter(suite, architecture, overridetype, component) - contents_writer.write_file() - return log_message - class BinaryContentsScanner(object): ''' diff --git a/tests/dbtest_contents.py b/tests/dbtest_contents.py index e3128161..0f23053c 100755 --- a/tests/dbtest_contents.py +++ b/tests/dbtest_contents.py @@ -3,8 +3,8 @@ from db_test import DBDakTestCase, fixture from daklib.dbconn import * -from daklib.contents import ContentsWriter, BinaryContentsScanner, \ - UnpackedSource, SourceContentsScanner +from daklib.contents import BinaryContentsWriter, BinaryContentsScanner, \ + UnpackedSource, SourceContentsScanner, SourceContentsWriter from os.path import normpath from sqlalchemy.exc import FlushError, IntegrityError @@ -131,9 +131,9 @@ class ContentsTestCase(DBDakTestCase): self.assertEqual(self.override['hello_sid_main_udeb'], \ self.otype['udeb'].overrides.one()) - def test_contentswriter(self): + def test_binarycontentswriter(self): ''' - Test the ContentsWriter class. + Test the BinaryContentsWriter class. ''' self.setup_suites() self.setup_architectures() @@ -142,7 +142,7 @@ class ContentsTestCase(DBDakTestCase): self.setup_overrides() self.binary['hello_2.2-1_i386'].contents.append(BinContents(file = '/usr/bin/hello')) self.session.commit() - cw = ContentsWriter(self.suite['squeeze'], self.arch['i386'], self.otype['deb']) + cw = BinaryContentsWriter(self.suite['squeeze'], self.arch['i386'], self.otype['deb']) self.assertEqual(['/usr/bin/hello python/hello\n'], \ cw.get_list()) # test formatline and sort order @@ -151,7 +151,7 @@ class ContentsTestCase(DBDakTestCase): # test output_filename self.assertEqual('tests/fixtures/ftp/dists/squeeze/Contents-i386.gz', \ normpath(cw.output_filename())) - cw = ContentsWriter(self.suite['squeeze'], self.arch['i386'], \ + cw = BinaryContentsWriter(self.suite['squeeze'], self.arch['i386'], \ self.otype['udeb'], self.comp['main']) self.assertEqual('tests/fixtures/ftp/dists/squeeze/main/Contents-i386.gz', \ normpath(cw.output_filename())) @@ -201,6 +201,22 @@ class ContentsTestCase(DBDakTestCase): SourceContentsScanner(source.source_id).scan() self.assertTrue(source.contents.count() > 0) + def test_sourcecontentswriter(self): + ''' + Test the SourceContentsWriter class. + ''' + self.setup_sources() + self.session.flush() + # remove newer package from sid because it disturbs the test + self.source['hello_2.2-2'].suites = [] + self.session.commit() + source = self.source['hello_2.2-1'] + SourceContentsScanner(source.source_id).scan() + cw = SourceContentsWriter(source.suites[0], source.poolfile.location.component) + result = cw.get_list() + self.assertEqual(8, len(result)) + self.assertTrue('debian/changelog\thello\n' in result) + def classes_to_clean(self): return [Override, Suite, BinContents, DBBinary, DBSource, Architecture, Section, \ OverrideType, Maintainer, Component, Priority, PoolFile]