X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=64437561c67a67e31646bc2b96d20097149af727;hb=e81d92555e88dbf275b465a306e8461e9f3991aa;hp=6948cf6c27902528030c17f0f8f602811a3b79dc;hpb=3fe6999ba255bd164e2c84c2c07ae5300ff5e8f7;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 6948cf6c..64437561 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -33,12 +33,14 @@ ################################################################################ +import apt_pkg import os from os.path import normpath import re import psycopg2 import traceback import commands +import signal try: # python >= 2.6 @@ -72,7 +74,7 @@ from sqlalchemy.orm.exc import NoResultFound # in the database from config import Config from textutils import fix_maintainer -from dak_exceptions import DBUpdateError, NoSourceFieldError +from dak_exceptions import DBUpdateError, NoSourceFieldError, FileExistsError # suppress some deprecation warnings in squeeze related to sqlalchemy import warnings @@ -487,6 +489,11 @@ __all__.append('BinContents') ################################################################################ +def subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + class DBBinary(ORMObject): def __init__(self, package = None, source = None, version = None, \ maintainer = None, architecture = None, poolfile = None, \ @@ -525,7 +532,8 @@ class DBBinary(ORMObject): package does not contain any regular file. ''' fullpath = self.poolfile.fullpath - dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE) + dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE, + preexec_fn = subprocess_setup) tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|') for member in tar.getmembers(): if not member.isdir(): @@ -856,8 +864,8 @@ class BuildQueue(object): # Check if we have a file of this name or this ID already for f in self.queuefiles: - if f.fileid is not None and f.fileid == poolfile.file_id or \ - f.poolfile.filename == poolfile_basename: + if (f.fileid is not None and f.fileid == poolfile.file_id) or \ + (f.poolfile is not None and f.poolfile.filename == poolfile_basename): # In this case, update the BuildQueueFile entry so we # don't remove it too early f.lastused = datetime.now() @@ -883,6 +891,9 @@ class BuildQueue(object): else: os.symlink(targetpath, queuepath) qf.fileid = poolfile.file_id + except FileExistsError: + if not poolfile.identical_to(queuepath): + raise except OSError: return None @@ -941,6 +952,9 @@ class BuildQueue(object): # Always copy files from policy queues as they might move around. import utils utils.copy(source, target) + except FileExistsError: + if not policyqueuefile.identical_to(target): + raise except OSError: return None @@ -1036,6 +1050,24 @@ class ChangePendingFile(object): def __repr__(self): return '' % self.change_pending_file_id + def identical_to(self, filename): + """ + compare size and hash with the given file + + @rtype: bool + @return: true if the given file has the same size and hash as this object; false otherwise + """ + st = os.stat(filename) + if self.size != st.st_size: + return False + + f = open(filename, "r") + sha256sum = apt_pkg.sha256sum(f) + if sha256sum != self.sha256sum: + return False + + return True + __all__.append('ChangePendingFile') ################################################################################ @@ -1352,6 +1384,17 @@ __all__.append('get_dscfiles') ################################################################################ +class ExternalOverride(ORMObject): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.package, self.key, self.value) + +__all__.append('ExternalOverride') + +################################################################################ + class PoolFile(ORMObject): def __init__(self, filename = None, location = None, filesize = -1, \ md5sum = None): @@ -1374,6 +1417,24 @@ class PoolFile(ORMObject): def not_null_constraints(self): return ['filename', 'md5sum', 'location'] + def identical_to(self, filename): + """ + compare size and hash with the given file + + @rtype: bool + @return: true if the given file has the same size and hash as this object; false otherwise + """ + st = os.stat(filename) + if self.filesize != st.st_size: + return False + + f = open(filename, "r") + sha256sum = apt_pkg.sha256sum(f) + if sha256sum != self.sha256sum: + return False + + return True + __all__.append('PoolFile') @session_wrapper @@ -2446,20 +2507,13 @@ def source_exists(source, source_version, suites = ["any"], session=None): q = session.query(DBSource).filter_by(source=source). \ filter(DBSource.version.in_([source_version, orig_source_version])) if suite != "any": - # source must exist in suite X, or in some other suite that's - # mapped to X, recursively... silent-maps are counted too, - # unreleased-maps aren't. - maps = cnf.ValueList("SuiteMappings")[:] - maps.reverse() - maps = [ m.split() for m in maps ] - maps = [ (x[1], x[2]) for x in maps - if x[0] == "map" or x[0] == "silent-map" ] - s = [suite] - for (from_, to) in maps: - if from_ in s and to not in s: - s.append(to) - - q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s))) + # source must exist in 'suite' or a suite that is enhanced by 'suite' + s = get_suite(suite, session) + enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances') + considered_suites = [ vc.reference for vc in enhances_vcs ] + considered_suites.append(s) + + q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites]))) if q.count() > 0: continue @@ -2581,6 +2635,15 @@ __all__.append('import_metadata_into_db') ################################################################################ +def split_uploaders(uploaders_list): + ''' + Split the Uploaders field into the individual uploaders and yield each of + them. Beware: email addresses might contain commas. + ''' + import re + for uploader in re.sub(">[ ]*,", ">\t", uploaders_list).split("\t"): + yield uploader.strip() + @session_wrapper def add_dsc_to_db(u, filename, session=None): entry = u.pkg.files[filename] @@ -2590,7 +2653,11 @@ def add_dsc_to_db(u, filename, session=None): source.source = u.pkg.dsc["source"] source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id - source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id + # If Changed-By isn't available, fall back to maintainer + if u.pkg.changes.has_key("changed-by"): + source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id + else: + source.changedby_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id source.install_date = datetime.now().date() @@ -2663,10 +2730,11 @@ def add_dsc_to_db(u, filename, session=None): session.add(df) # Add the src_uploaders to the DB + session.flush() + session.refresh(source) source.uploaders = [source.maintainer] if u.pkg.dsc.has_key("uploaders"): - for up in u.pkg.dsc["uploaders"].replace(">, ", ">\t").split("\t"): - up = up.strip() + for up in split_uploaders(u.pkg.dsc["uploaders"]): source.uploaders.append(get_or_set_maintainer(up, session)) session.flush() @@ -2861,6 +2929,12 @@ class Suite(ORMObject): return session.query(DBSource).filter_by(source = source). \ with_parent(self) + def get_overridesuite(self): + if self.overridesuite is None: + return self + else: + return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one() + __all__.append('Suite') @session_wrapper @@ -3136,7 +3210,9 @@ __all__.append('VersionCheck') def get_version_checks(suite_name, check = None, session = None): suite = get_suite(suite_name, session) if not suite: - return None + # Make sure that what we return is iterable so that list comprehensions + # involving this don't cause a traceback + return [] q = session.query(VersionCheck).filter_by(suite=suite) if check: q = q.filter_by(check=check) @@ -3184,6 +3260,7 @@ class DBConn(object): 'changes_pending_source_files', 'changes_pool_files', 'dsc_files', + 'external_overrides', 'extra_src_references', 'files', 'fingerprint', @@ -3318,6 +3395,13 @@ class DBConn(object): poolfile_id = self.tbl_dsc_files.c.file, poolfile = relation(PoolFile))) + mapper(ExternalOverride, self.tbl_external_overrides, + properties = dict( + suite_id = self.tbl_external_overrides.c.suite, + suite = relation(Suite), + component_id = self.tbl_external_overrides.c.component, + component = relation(Component))) + mapper(PoolFile, self.tbl_files, properties = dict(file_id = self.tbl_files.c.id, filesize = self.tbl_files.c.size,