X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fcontents.py;h=75fb5e59b96d7c06a54fcc2a0b2dc4c2b2103ae7;hb=391f5ec09a119131dc846b796ca791f4cecc69e4;hp=e808da6234b97194b6611a7e798dd185921e6425;hpb=dfb9c4b902ab992e102d5496a2ef2c1e2442b742;p=dak.git diff --git a/daklib/contents.py b/daklib/contents.py old mode 100755 new mode 100644 index e808da62..75fb5e59 --- a/daklib/contents.py +++ b/daklib/contents.py @@ -31,11 +31,10 @@ from daklib.filewriter import BinaryContentsFileWriter, SourceContentsFileWriter from multiprocessing import Pool from shutil import rmtree -from subprocess import Popen, PIPE, check_call from tempfile import mkdtemp +import daklib.daksubprocess import os.path -import signal class BinaryContentsWriter(object): ''' @@ -65,15 +64,23 @@ class BinaryContentsWriter(object): 'type': self.overridetype.overridetype, } - sql = ''' -with + sql_create_temp = ''' +create temp table newest_binaries ( + id integer primary key, + package text); + +create index newest_binaries_by_package on newest_binaries (package); -newest_binaries as - (select distinct on (package) id, package from binaries +insert into newest_binaries (id, package) + select distinct on (package) id, package from binaries where type = :type and (architecture = :arch_all or architecture = :arch) and id in (select bin from bin_associations where suite = :suite) - order by package, version desc), + order by package, version desc;''' + self.session.execute(sql_create_temp, params=params) + + sql = ''' +with unique_override as (select o.package, s.section @@ -115,6 +122,7 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package Returns a writer object. ''' values = { + 'archive': self.suite.archive.path, 'suite': self.suite.suite_name, 'component': self.component.component_name, 'debtype': self.overridetype.overridetype, @@ -126,14 +134,9 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package ''' Returns the header for the Contents files as a string. ''' - header_file = None - try: - filename = os.path.join(Config()['Dir::Templates'], 'contents') - header_file = open(filename) + filename = os.path.join(Config()['Dir::Templates'], 'contents') + with open(filename) as header_file: return header_file.read() - finally: - if header_file: - header_file.close() def write_file(self): ''' @@ -165,16 +168,22 @@ class SourceContentsWriter(object): 'component_id': self.component.component_id, } - sql = ''' -with - newest_sources as - (select distinct on (source) s.id, s.source from source s - join files f on f.id = s.file - join location l on l.id = f.location + sql_create_temp = ''' +create temp table newest_sources ( + id integer primary key, + source text); + +create index sources_binaries_by_source on newest_sources (source); + +insert into newest_sources (id, source) + select distinct on (source) s.id, s.source from source s + join files_archive_map af on s.file = af.file_id where s.id in (select source from src_associations where suite = :suite_id) - and l.component = :component_id - order by source, version desc) + and af.component_id = :component_id + order by source, version desc;''' + self.session.execute(sql_create_temp, params=params) + sql = ''' select sc.file, string_agg(s.source, ',' order by s.source) as pkglist from newest_sources s, src_contents sc where s.id = sc.source_id group by sc.file''' @@ -208,6 +217,7 @@ select sc.file, string_agg(s.source, ',' order by s.source) as pkglist Returns a writer object. ''' values = { + 'archive': self.suite.archive.path, 'suite': self.suite.suite_name, 'component': self.component.component_name } @@ -238,6 +248,7 @@ def binary_helper(suite_id, arch_id, overridetype_id, component_id): overridetype.overridetype, component.component_name] contents_writer = BinaryContentsWriter(suite, architecture, overridetype, component) contents_writer.write_file() + session.close() return log_message def source_helper(suite_id, component_id): @@ -251,6 +262,7 @@ def source_helper(suite_id, component_id): log_message = [suite.suite_name, 'source', component.component_name] contents_writer = SourceContentsWriter(suite, component) contents_writer.write_file() + session.close() return log_message class ContentsWriter(object): @@ -266,7 +278,7 @@ class ContentsWriter(object): class_.logger.log(result) @classmethod - def write_all(class_, logger, suite_names = [], component_names = [], force = False): + def write_all(class_, logger, archive_names = [], suite_names = [], component_names = [], force = False): ''' Writes all Contents files for suites in list suite_names which defaults to all 'touchable' suites if not specified explicitely. Untouchable @@ -275,16 +287,23 @@ class ContentsWriter(object): class_.logger = logger session = DBConn().session() suite_query = session.query(Suite) + if len(archive_names) > 0: + suite_query = suite_query.join(Suite.archive).filter(Archive.archive_name.in_(archive_names)) if len(suite_names) > 0: suite_query = suite_query.filter(Suite.suite_name.in_(suite_names)) component_query = session.query(Component) if len(component_names) > 0: component_query = component_query.filter(Component.component_name.in_(component_names)) if not force: - suite_query = suite_query.filter_by(untouchable = False) + suite_query = suite_query.filter(Suite.untouchable == False) deb_id = get_override_type('deb', session).overridetype_id udeb_id = get_override_type('udeb', session).overridetype_id pool = Pool() + + # Lock tables so that nobody can change things underneath us + session.execute("LOCK TABLE bin_contents IN SHARE MODE") + session.execute("LOCK TABLE src_contents IN SHARE MODE") + for suite in suite_query: suite_id = suite.suite_id for component in component_query: @@ -363,26 +382,21 @@ def binary_scan_helper(binary_id): scanner = BinaryContentsScanner(binary_id) scanner.scan() - -def subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - class UnpackedSource(object): ''' UnpackedSource extracts a source package into a temporary location and gives you some convinient function for accessing it. ''' - def __init__(self, dscfilename): + def __init__(self, dscfilename, tmpbasedir=None): ''' The dscfilename is a name of a DSC file that will be extracted. ''' - temp_directory = mkdtemp(dir = Config()['Dir::TempPath']) + basedir = tmpbasedir if tmpbasedir else Config()['Dir::TempPath'] + temp_directory = mkdtemp(dir = basedir) self.root_directory = os.path.join(temp_directory, 'root') command = ('dpkg-source', '--no-copy', '--no-check', '-q', '-x', dscfilename, self.root_directory) - check_call(command, preexec_fn = subprocess_setup) + daklib.daksubprocess.check_call(command) def get_root_directory(self): ''' @@ -485,4 +499,3 @@ def source_scan_helper(source_id): scanner.scan() except Exception as e: print e -