X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=1972e412af81e4a5bad852c30a14f13405945165;hb=790bd8d1f10012d547519a1ddb55464b969b54ef;hp=5b5dc5bc707f375a5027cf75777e47ec41b23ee7;hpb=116d60e8e849644ace31f5d5c05465e4f2d83494;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py old mode 100755 new mode 100644 index 5b5dc5bc..1972e412 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -42,6 +42,8 @@ import traceback import commands import signal +from daklib.gpg import SignedFile + try: # python >= 2.6 import json @@ -81,6 +83,9 @@ import warnings warnings.filterwarnings('ignore', \ "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \ SADeprecationWarning) +warnings.filterwarnings('ignore', \ + "Predicate of partial index .* ignored during reflection", \ + SAWarning) ################################################################################ @@ -107,11 +112,11 @@ class DebVersion(UserDefinedType): return None sa_major_version = sqlalchemy.__version__[0:3] -if sa_major_version in ["0.5", "0.6"]: +if sa_major_version in ["0.5", "0.6", "0.7"]: from sqlalchemy.databases import postgres postgres.ischema_names['debversion'] = DebVersion else: - raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py") + raise Exception("dak only ported to SQLA versions 0.5 to 0.7. See daklib/dbconn.py") ################################################################################ @@ -477,6 +482,19 @@ __all__.append('get_archive') ################################################################################ +class ArchiveFile(object): + def __init__(self, archive=None, component=None, file=None): + self.archive = archive + self.component = component + self.file = file + @property + def path(self): + return os.path.join(self.archive.path, 'pool', self.component.component_name, self.file.filename) + +__all__.append('ArchiveFile') + +################################################################################ + class BinContents(ORMObject): def __init__(self, file = None, binary = None): self.file = file @@ -497,7 +515,7 @@ def subprocess_setup(): class DBBinary(ORMObject): def __init__(self, package = None, source = None, version = None, \ maintainer = None, architecture = None, poolfile = None, \ - binarytype = 'deb'): + binarytype = 'deb', fingerprint=None): self.package = package self.source = source self.version = version @@ -505,6 +523,7 @@ class DBBinary(ORMObject): self.architecture = architecture self.poolfile = poolfile self.binarytype = binarytype + self.fingerprint = fingerprint @property def pkid(self): @@ -555,10 +574,10 @@ class DBBinary(ORMObject): @rtype: text @return: stanza text of the control section. ''' - import apt_inst + import utils fullpath = self.poolfile.fullpath deb_file = open(fullpath, 'r') - stanza = apt_inst.debExtractControl(deb_file) + stanza = utils.deb_extract_control(deb_file) deb_file.close() return stanza @@ -815,7 +834,7 @@ class BuildQueue(object): Logger.log(["I: Removing %s from the queue" % o.fullpath]) os.unlink(o.fullpath) killdb = True - except OSError, e: + except OSError as e: # If it wasn't there, don't worry if e.errno == ENOENT: killdb = True @@ -875,7 +894,6 @@ class BuildQueue(object): # Prepare BuildQueueFile object qf = BuildQueueFile() qf.build_queue_id = self.queue_id - qf.lastused = datetime.now() qf.filename = poolfile_basename targetpath = poolfile.fullpath @@ -1132,6 +1150,19 @@ def get_component(component, session=None): __all__.append('get_component') +@session_wrapper +def get_component_names(session=None): + """ + Returns list of strings of component names. + + @rtype: list + @return: list of strings of component names + """ + + return [ x.component_name for x in session.query(Component).all() ] + +__all__.append('get_component_names') + ################################################################################ class DBConfig(object): @@ -1405,17 +1436,30 @@ class PoolFile(ORMObject): @property def fullpath(self): - return os.path.join(self.location.path, self.filename) + session = DBConn().session().object_session(self) + af = session.query(ArchiveFile).join(Archive).filter(ArchiveFile.file == self).first() + return af.path + + @property + def component(self): + session = DBConn().session().object_session(self) + component_id = session.query(ArchiveFile.component_id).filter(ArchiveFile.file == self) \ + .group_by(ArchiveFile.component_id).one() + return session.query(Component).get(component_id) + + @property + def basename(self): + return os.path.basename(self.filename) def is_valid(self, filesize = -1, md5sum = None): return self.filesize == long(filesize) and self.md5sum == md5sum def properties(self): return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \ - 'sha256sum', 'location', 'source', 'binary', 'last_used'] + 'sha256sum', 'source', 'binary', 'last_used'] def not_null_constraints(self): - return ['filename', 'md5sum', 'location'] + return ['filename', 'md5sum'] def identical_to(self, filename): """ @@ -1670,7 +1714,7 @@ class Keyring(object): key = None signingkey = False - for line in k.xreadlines(): + for line in k: field = line.split(":") if field[0] == "pub": key = field[4] @@ -1777,6 +1821,34 @@ def get_keyring(keyring, session=None): __all__.append('get_keyring') +@session_wrapper +def get_active_keyring_paths(session=None): + """ + @rtype: list + @return: list of active keyring paths + """ + return [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).order_by(desc(Keyring.priority)).all() ] + +__all__.append('get_active_keyring_paths') + +@session_wrapper +def get_primary_keyring_path(session=None): + """ + Get the full path to the highest priority active keyring + + @rtype: str or None + @return: path to the active keyring with the highest priority or None if no + keyring is configured + """ + keyrings = get_active_keyring_paths() + + if len(keyrings) > 0: + return keyrings[0] + else: + return None + +__all__.append('get_primary_keyring_path') + ################################################################################ class KeyringACLMap(object): @@ -2201,6 +2273,31 @@ __all__.append('get_policy_queue_from_path') ################################################################################ +class PolicyQueueUpload(object): + def __cmp__(self, other): + ret = cmp(self.changes.source, other.changes.source) + if ret == 0: + ret = apt_pkg.version_compare(self.changes.version, other.changes.version) + if ret == 0: + if self.source is not None and other.source is None: + ret = -1 + elif self.source is None and other.source is not None: + ret = 1 + if ret == 0: + ret = cmp(self.changes.changesname, other.changes.changesname) + return ret + +__all__.append('PolicyQueueUpload') + +################################################################################ + +class PolicyQueueByhandFile(object): + pass + +__all__.append('PolicyQueueByhandFile') + +################################################################################ + class Priority(ORMObject): def __init__(self, priority = None, level = None): self.priority = priority @@ -2360,69 +2457,16 @@ __all__.append('SrcContents') ################################################################################ -from debian.debfile import Deb822 - -# Temporary Deb822 subclass to fix bugs with : handling; see #597249 -class Dak822(Deb822): - def _internal_parser(self, sequence, fields=None): - # The key is non-whitespace, non-colon characters before any colon. - key_part = r"^(?P[^: \t\n\r\f\v]+)\s*:\s*" - single = re.compile(key_part + r"(?P\S.*?)\s*$") - multi = re.compile(key_part + r"$") - multidata = re.compile(r"^\s(?P.+?)\s*$") - - wanted_field = lambda f: fields is None or f in fields - - if isinstance(sequence, basestring): - sequence = sequence.splitlines() - - curkey = None - content = "" - for line in self.gpg_stripped_paragraph(sequence): - m = single.match(line) - if m: - if curkey: - self[curkey] = content - - if not wanted_field(m.group('key')): - curkey = None - continue - - curkey = m.group('key') - content = m.group('data') - continue - - m = multi.match(line) - if m: - if curkey: - self[curkey] = content - - if not wanted_field(m.group('key')): - curkey = None - continue - - curkey = m.group('key') - content = "" - continue - - m = multidata.match(line) - if m: - content += '\n' + line # XXX not m.group('data')? - continue - - if curkey: - self[curkey] = content - - class DBSource(ORMObject): def __init__(self, source = None, version = None, maintainer = None, \ - changedby = None, poolfile = None, install_date = None): + changedby = None, poolfile = None, install_date = None, fingerprint = None): self.source = source self.version = version self.maintainer = maintainer self.changedby = changedby self.poolfile = poolfile self.install_date = install_date + self.fingerprint = fingerprint @property def pkid(self): @@ -2435,7 +2479,7 @@ class DBSource(ORMObject): def not_null_constraints(self): return ['source', 'version', 'install_date', 'maintainer', \ - 'changedby', 'poolfile', 'install_date'] + 'changedby', 'poolfile'] def read_control_fields(self): ''' @@ -2445,11 +2489,16 @@ class DBSource(ORMObject): @return: fields is the dsc information in a dictionary form ''' fullpath = self.poolfile.fullpath - fields = Dak822(open(self.poolfile.fullpath, 'r')) + contents = open(fullpath, 'r').read() + signed_file = SignedFile(contents, keyrings=[], require_signature=False) + fields = apt_pkg.TagSection(signed_file.contents) return fields metadata = association_proxy('key', 'value') + def get_component_name(self): + return self.poolfile.location.component.component_name + def scan_contents(self): ''' Returns a set of names for non directories. The path names are @@ -2509,11 +2558,12 @@ def source_exists(source, source_version, suites = ["any"], session=None): if suite != "any": # source must exist in 'suite' or a suite that is enhanced by 'suite' s = get_suite(suite, session) - enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances') - considered_suites = [ vc.reference for vc in enhances_vcs ] - considered_suites.append(s) + if s: + enhances_vcs = session.query(VersionCheck).filter(VersionCheck.suite==s).filter_by(check='Enhances') + considered_suites = [ vc.reference for vc in enhances_vcs ] + considered_suites.append(s) - q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites]))) + q = q.filter(DBSource.suites.any(Suite.suite_id.in_([s.suite_id for s in considered_suites]))) if q.count() > 0: continue @@ -2776,10 +2826,25 @@ def add_deb_to_db(u, filename, session=None): # Find source id bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session) + + # If we couldn't find anything and the upload contains Arch: source, + # fall back to trying the source package, source version uploaded + # This maintains backwards compatibility with previous dak behaviour + # and deals with slightly broken binary debs which don't properly + # declare their source package name + if len(bin_sources) == 0: + if u.pkg.changes["architecture"].has_key("source") \ + and u.pkg.dsc.has_key("source") and u.pkg.dsc.has_key("version"): + bin_sources = get_sources_from_name(u.pkg.dsc["source"], u.pkg.dsc["version"], session=session) + + # If we couldn't find a source here, we reject + # TODO: Fix this so that it doesn't kill process-upload and instead just + # performs a reject. To be honest, we should probably spot this + # *much* earlier than here if len(bin_sources) != 1: - raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \ + raise NoSourceFieldError("Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \ (bin.package, bin.version, entry["architecture"], - filename, bin.binarytype, u.pkg.changes["fingerprint"]) + filename, bin.binarytype, u.pkg.changes["fingerprint"])) bin.source_id = bin_sources[0].source_id @@ -2787,9 +2852,9 @@ def add_deb_to_db(u, filename, session=None): for srcname, version in entry["built-using"]: exsources = get_sources_from_name(srcname, version, session=session) if len(exsources) != 1: - raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \ + raise NoSourceFieldError("Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \ (srcname, version, bin.package, bin.version, entry["architecture"], - filename, bin.binarytype, u.pkg.changes["fingerprint"]) + filename, bin.binarytype, u.pkg.changes["fingerprint"])) bin.extra_sources.append(exsources[0]) @@ -2935,6 +3000,10 @@ class Suite(ORMObject): else: return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one() + @property + def path(self): + return os.path.join(self.archive.path, 'dists', self.suite_name) + __all__.append('Suite') @session_wrapper @@ -2964,11 +3033,11 @@ __all__.append('get_suite') ################################################################################ -# TODO: should be removed because the implementation is too trivial @session_wrapper def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): """ - Returns list of Architecture objects for given C{suite} name + Returns list of Architecture objects for given C{suite} name. The list is + empty if suite does not exist. @type suite: str @param suite: Suite name to search for @@ -2989,48 +3058,15 @@ def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): @return: list of Architecture objects for the given name (may be empty) """ - return get_suite(suite, session).get_architectures(skipsrc, skipall) + try: + return get_suite(suite, session).get_architectures(skipsrc, skipall) + except AttributeError: + return [] __all__.append('get_suite_architectures') ################################################################################ -class SuiteSrcFormat(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.suite_id, self.src_format_id) - -__all__.append('SuiteSrcFormat') - -@session_wrapper -def get_suite_src_formats(suite, session=None): - """ - Returns list of allowed SrcFormat for C{suite}. - - @type suite: str - @param suite: Suite name to search for - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: the list of allowed source formats for I{suite} - """ - - q = session.query(SrcFormat) - q = q.join(SuiteSrcFormat) - q = q.join(Suite).filter_by(suite_name=suite) - q = q.order_by('format_name') - - return q.all() - -__all__.append('get_suite_src_formats') - -################################################################################ - class Uid(ORMObject): def __init__(self, uid = None, name = None): self.uid = uid @@ -3263,6 +3299,7 @@ class DBConn(object): 'external_overrides', 'extra_src_references', 'files', + 'files_archive_map', 'fingerprint', 'keyrings', 'keyring_acl_map', @@ -3274,6 +3311,9 @@ class DBConn(object): 'override', 'override_type', 'policy_queue', + 'policy_queue_upload', + 'policy_queue_upload_binaries_map', + 'policy_queue_byhand_file', 'priority', 'section', 'source', @@ -3298,7 +3338,6 @@ class DBConn(object): 'any_associations_source', 'bin_associations_binaries', 'binaries_suite_arch', - 'binfiles_suite_component_arch', 'changelogs', 'file_arch_suite', 'newest_all_associations', @@ -3328,16 +3367,22 @@ class DBConn(object): mapper(Architecture, self.tbl_architecture, properties = dict(arch_id = self.tbl_architecture.c.id, suites = relation(Suite, secondary=self.tbl_suite_architectures, - order_by='suite_name', - backref=backref('architectures', order_by='arch_string'))), + order_by=self.tbl_suite.c.suite_name, + backref=backref('architectures', order_by=self.tbl_architecture.c.arch_string))), extension = validator) mapper(Archive, self.tbl_archive, properties = dict(archive_id = self.tbl_archive.c.id, archive_name = self.tbl_archive.c.name)) + mapper(ArchiveFile, self.tbl_files_archive_map, + properties = dict(archive = relation(Archive, backref='files'), + component = relation(Component), + file = relation(PoolFile, backref='archives'))) + mapper(BuildQueue, self.tbl_build_queue, - properties = dict(queue_id = self.tbl_build_queue.c.id)) + properties = dict(queue_id = self.tbl_build_queue.c.id, + suite = relation(Suite, primaryjoin=(self.tbl_build_queue.c.suite_id==self.tbl_suite.c.id)))) mapper(BuildQueueFile, self.tbl_build_queue_files, properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'), @@ -3404,13 +3449,7 @@ class DBConn(object): mapper(PoolFile, self.tbl_files, properties = dict(file_id = self.tbl_files.c.id, - filesize = self.tbl_files.c.size, - location_id = self.tbl_files.c.location, - location = relation(Location, - # using lazy='dynamic' in the back - # reference because we have A LOT of - # files in one location - backref=backref('files', lazy='dynamic'))), + filesize = self.tbl_files.c.size), extension = validator) mapper(Fingerprint, self.tbl_fingerprint, @@ -3526,6 +3565,21 @@ class DBConn(object): mapper(PolicyQueue, self.tbl_policy_queue, properties = dict(policy_queue_id = self.tbl_policy_queue.c.id)) + mapper(PolicyQueueUpload, self.tbl_policy_queue_upload, + properties = dict( + changes = relation(DBChange), + policy_queue = relation(PolicyQueue, backref='uploads'), + target_suite = relation(Suite), + source = relation(DBSource), + binaries = relation(DBBinary, secondary=self.tbl_policy_queue_upload_binaries_map), + )) + + mapper(PolicyQueueByhandFile, self.tbl_policy_queue_byhand_file, + properties = dict( + upload = relation(PolicyQueueUpload, backref='byhand'), + ) + ) + mapper(Priority, self.tbl_priority, properties = dict(priority_id = self.tbl_priority.c.id)) @@ -3563,15 +3617,12 @@ class DBConn(object): properties = dict(suite_id = self.tbl_suite.c.id, policy_queue = relation(PolicyQueue), copy_queues = relation(BuildQueue, - secondary=self.tbl_suite_build_queue_copy)), + secondary=self.tbl_suite_build_queue_copy), + srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats, + backref=backref('suites', lazy='dynamic')), + archive = relation(Archive, backref='suites')), extension = validator) - mapper(SuiteSrcFormat, self.tbl_suite_src_formats, - properties = dict(suite_id = self.tbl_suite_src_formats.c.suite, - suite = relation(Suite, backref='suitesrcformats'), - src_format_id = self.tbl_suite_src_formats.c.src_format, - src_format = relation(SrcFormat))) - mapper(Uid, self.tbl_uid, properties = dict(uid_id = self.tbl_uid.c.id, fingerprint = relation(Fingerprint)), @@ -3663,15 +3714,21 @@ class DBConn(object): sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak - self.db_pg = create_engine(connstr, **engine_args) - self.db_meta = MetaData() - self.db_meta.bind = self.db_pg - self.db_smaker = sessionmaker(bind=self.db_pg, - autoflush=True, - autocommit=False) + try: + self.db_pg = create_engine(connstr, **engine_args) + self.db_meta = MetaData() + self.db_meta.bind = self.db_pg + self.db_smaker = sessionmaker(bind=self.db_pg, + autoflush=True, + autocommit=False) + + self.__setuptables() + self.__setupmappers() + + except OperationalError as e: + import utils + utils.fubar("Cannot connect to database (%s)" % str(e)) - self.__setuptables() - self.__setupmappers() self.pid = os.getpid() def session(self, work_mem = 0):