X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=988309361b3bef1fa2295c8184bedc8f60eddc31;hb=7c73ca0018de48b6366b0e26c37439f20ff4b2d5;hp=293f4dcb799e7abcf0587194dfa593925e1df58d;hpb=e2a848e480d8aabbfef4620a571088030af1013f;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 293f4dcb..98830936 100644 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -34,13 +34,13 @@ ################################################################################ import apt_pkg +import daklib.daksubprocess import os from os.path import normpath import re import psycopg2 +import subprocess import traceback -import commands -import signal try: # python >= 2.6 @@ -52,7 +52,6 @@ except: from datetime import datetime, timedelta from errno import ENOENT from tempfile import mkstemp, mkdtemp -from subprocess import Popen, PIPE from tarfile import TarFile from inspect import getargspec @@ -110,11 +109,11 @@ class DebVersion(UserDefinedType): return None sa_major_version = sqlalchemy.__version__[0:3] -if sa_major_version in ["0.5", "0.6", "0.7"]: +if sa_major_version in ["0.5", "0.6", "0.7", "0.8", "0.9"]: from sqlalchemy.databases import postgres postgres.ischema_names['debversion'] = DebVersion else: - raise Exception("dak only ported to SQLA versions 0.5 to 0.7. See daklib/dbconn.py") + raise Exception("dak only ported to SQLA versions 0.5 to 0.9. See daklib/dbconn.py") ################################################################################ @@ -311,7 +310,7 @@ class ORMObject(object): return object_session(self) def clone(self, session = None): - ''' + """ Clones the current object in a new session and returns the new clone. A fresh session is created if the optional session parameter is not provided. The function will fail if a session is provided and has @@ -324,8 +323,8 @@ class ORMObject(object): WARNING: Only persistent (committed) objects can be cloned. Changes made to the original object that are not committed yet will get lost. The session of the new object will always be rolled back to avoid - ressource leaks. - ''' + resource leaks. + """ if self.session() is None: raise RuntimeError( \ @@ -369,6 +368,20 @@ validator = Validator() ################################################################################ +class ACL(ORMObject): + def __repr__(self): + return "".format(self.name) + +__all__.append('ACL') + +class ACLPerSource(ORMObject): + def __repr__(self): + return "".format(self.acl.name, self.fingerprint.fingerprint, self.source, self.reason) + +__all__.append('ACLPerSource') + +################################################################################ + class Architecture(ORMObject): def __init__(self, arch_string = None, description = None): self.arch_string = arch_string @@ -419,27 +432,6 @@ def get_architecture(architecture, session=None): __all__.append('get_architecture') -# TODO: should be removed because the implementation is too trivial -@session_wrapper -def get_architecture_suites(architecture, session=None): - """ - Returns list of Suite objects for given C{architecture} name - - @type architecture: str - @param architecture: Architecture name to search for - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of Suite objects for the given name (may be empty) - """ - - return get_architecture(architecture, session).suites - -__all__.append('get_architecture_suites') - ################################################################################ class Archive(object): @@ -505,11 +497,6 @@ __all__.append('BinContents') ################################################################################ -def subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - class DBBinary(ORMObject): def __init__(self, package = None, source = None, version = None, \ maintainer = None, architecture = None, poolfile = None, \ @@ -546,8 +533,8 @@ class DBBinary(ORMObject): package does not contain any regular file. ''' fullpath = self.poolfile.fullpath - dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE, - preexec_fn = subprocess_setup) + dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', fullpath) + dpkg = daklib.daksubprocess.Popen(dpkg_cmd, stdout=subprocess.PIPE) tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|') for member in tar.getmembers(): if not member.isdir(): @@ -585,7 +572,6 @@ class DBBinary(ORMObject): @rtype: dict @return: fields of the control section as a dictionary. ''' - import apt_pkg stanza = self.read_control() return apt_pkg.TagSection(stanza) @@ -642,28 +628,6 @@ __all__.append('get_component_by_package_suite') ################################################################################ -class BinaryACL(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.binary_acl_id - -__all__.append('BinaryACL') - -################################################################################ - -class BinaryACLMap(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.binary_acl_map_id - -__all__.append('BinaryACLMap') - -################################################################################ - class BuildQueue(object): def __init__(self, *args, **kwargs): pass @@ -1038,7 +1002,9 @@ class PoolFile(ORMObject): @property def fullpath(self): session = DBConn().session().object_session(self) - af = session.query(ArchiveFile).join(Archive).filter(ArchiveFile.file == self).first() + af = session.query(ArchiveFile).join(Archive) \ + .filter(ArchiveFile.file == self) \ + .order_by(Archive.tainted.desc()).first() return af.path @property @@ -1226,7 +1192,7 @@ class Keyring(object): k = os.popen(self.gpg_invocation % keyring, "r") key = None - signingkey = False + need_fingerprint = False for line in k: field = line.split(":") @@ -1237,18 +1203,16 @@ class Keyring(object): if "@" in addr: self.keys[key]["email"] = addr self.keys[key]["name"] = name - self.keys[key]["fingerprints"] = [] - signingkey = True - elif key and field[0] == "sub" and len(field) >= 12: - signingkey = ("s" in field[11]) + need_fingerprint = True elif key and field[0] == "uid": (name, addr) = self.parse_address(field[9]) if "email" not in self.keys[key] and "@" in addr: self.keys[key]["email"] = addr self.keys[key]["name"] = name - elif signingkey and field[0] == "fpr": - self.keys[key]["fingerprints"].append(field[9]) + elif need_fingerprint and field[0] == "fpr": + self.keys[key]["fingerprints"] = [field[9]] self.fpr_lookup[field[9]] = key + need_fingerprint = False def import_users_from_ldap(self, session): import ldap @@ -1256,8 +1220,22 @@ class Keyring(object): LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"] LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"] + ca_cert_file = cnf.get('Import-LDAP-Fingerprints::CACertFile') l = ldap.open(LDAPServer) + + if ca_cert_file: + # TODO: This should request a new context and use + # connection-specific options (i.e. "l.set_option(...)") + + # Request a new TLS context. If there was already one, libldap + # would not change the TLS options (like which CAs to trust). + #l.set_option(ldap.OPT_X_TLS_NEWCTX, True) + ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_HARD) + #ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, None) + ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, ca_cert_file) + l.start_tls_s() + l.simple_bind_s("","") Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]), @@ -1365,17 +1343,6 @@ __all__.append('get_primary_keyring_path') ################################################################################ -class KeyringACLMap(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.keyring_acl_map_id - -__all__.append('KeyringACLMap') - -################################################################################ - class DBChange(object): def __init__(self, *args, **kwargs): pass @@ -1493,7 +1460,7 @@ class NewComment(object): __all__.append('NewComment') @session_wrapper -def has_new_comment(package, version, session=None): +def has_new_comment(policy_queue, package, version, session=None): """ Returns true if the given combination of C{package}, C{version} has a comment. @@ -1511,7 +1478,7 @@ def has_new_comment(package, version, session=None): @return: true/false """ - q = session.query(NewComment) + q = session.query(NewComment).filter_by(policy_queue=policy_queue) q = q.filter_by(package=package) q = q.filter_by(version=version) @@ -1520,7 +1487,7 @@ def has_new_comment(package, version, session=None): __all__.append('has_new_comment') @session_wrapper -def get_new_comments(package=None, version=None, comment_id=None, session=None): +def get_new_comments(policy_queue, package=None, version=None, comment_id=None, session=None): """ Returns (possibly empty) list of NewComment objects for the given parameters @@ -1542,7 +1509,7 @@ def get_new_comments(package=None, version=None, comment_id=None, session=None): @return: A (possibly empty) list of NewComment objects will be returned """ - q = session.query(NewComment) + q = session.query(NewComment).filter_by(policy_queue=policy_queue) if package is not None: q = q.filter_by(package=package) if version is not None: q = q.filter_by(version=version) if comment_id is not None: q = q.filter_by(comment_id=comment_id) @@ -1869,6 +1836,26 @@ __all__.append('get_sections') ################################################################################ +class SignatureHistory(ORMObject): + @classmethod + def from_signed_file(cls, signed_file): + """signature history entry from signed file + + @type signed_file: L{daklib.gpg.SignedFile} + @param signed_file: signed file + + @rtype: L{SignatureHistory} + """ + self = cls() + self.fingerprint = signed_file.primary_fingerprint + self.signature_timestamp = signed_file.signature_timestamp + self.contents_sha1 = signed_file.contents_sha1() + return self + +__all__.append('SignatureHistory') + +################################################################################ + class SrcContents(ORMObject): def __init__(self, file = None, source = None): self.file = file @@ -2157,17 +2144,6 @@ __all__.append('import_metadata_into_db') ################################################################################ -class SourceACL(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.source_acl_id - -__all__.append('SourceACL') - -################################################################################ - class SrcFormat(object): def __init__(self, *args, **kwargs): pass @@ -2417,17 +2393,6 @@ __all__.append('get_uid_from_fingerprint') ################################################################################ -class UploadBlock(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.source, self.upload_block_id) - -__all__.append('UploadBlock') - -################################################################################ - class MetadataKey(ORMObject): def __init__(self, key = None): self.key = key @@ -2551,18 +2516,21 @@ class DBConn(object): def __setuptables(self): tables = ( + 'acl', + 'acl_architecture_map', + 'acl_fingerprint_map', + 'acl_per_source', 'architecture', 'archive', 'bin_associations', 'bin_contents', 'binaries', 'binaries_metadata', - 'binary_acl', - 'binary_acl_map', 'build_queue', 'changelogs_text', 'changes', 'component', + 'component_suite', 'config', 'dsc_files', 'external_overrides', @@ -2571,7 +2539,6 @@ class DBConn(object): 'files_archive_map', 'fingerprint', 'keyrings', - 'keyring_acl_map', 'maintainer', 'metadata_keys', 'new_comments', @@ -2584,19 +2551,19 @@ class DBConn(object): 'policy_queue_byhand_file', 'priority', 'section', + 'signature_history', 'source', - 'source_acl', 'source_metadata', 'src_associations', 'src_contents', 'src_format', 'src_uploaders', 'suite', + 'suite_acl_map', 'suite_architectures', 'suite_build_queue_copy', 'suite_src_formats', 'uid', - 'upload_blocks', 'version_check', ) @@ -2639,6 +2606,21 @@ class DBConn(object): backref=backref('architectures', order_by=self.tbl_architecture.c.arch_string))), extension = validator) + mapper(ACL, self.tbl_acl, + properties = dict( + architectures = relation(Architecture, secondary=self.tbl_acl_architecture_map, collection_class=set), + fingerprints = relation(Fingerprint, secondary=self.tbl_acl_fingerprint_map, collection_class=set), + match_keyring = relation(Keyring, primaryjoin=(self.tbl_acl.c.match_keyring_id == self.tbl_keyrings.c.id)), + per_source = relation(ACLPerSource, collection_class=set), + )) + + mapper(ACLPerSource, self.tbl_acl_per_source, + properties = dict( + acl = relation(ACL), + fingerprint = relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.fingerprint_id == self.tbl_fingerprint.c.id)), + created_by = relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.created_by_id == self.tbl_fingerprint.c.id)), + )) + mapper(Archive, self.tbl_archive, properties = dict(archive_id = self.tbl_archive.c.id, archive_name = self.tbl_archive.c.name)) @@ -2676,14 +2658,6 @@ class DBConn(object): collection_class=attribute_mapped_collection('key'))), extension = validator) - mapper(BinaryACL, self.tbl_binary_acl, - properties = dict(binary_acl_id = self.tbl_binary_acl.c.id)) - - mapper(BinaryACLMap, self.tbl_binary_acl_map, - properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id, - fingerprint = relation(Fingerprint, backref="binary_acl_map"), - architecture = relation(Architecture))) - mapper(Component, self.tbl_component, properties = dict(component_id = self.tbl_component.c.id, component_name = self.tbl_component.c.name), @@ -2717,13 +2691,13 @@ class DBConn(object): uid = relation(Uid), keyring_id = self.tbl_fingerprint.c.keyring, keyring = relation(Keyring), - source_acl = relation(SourceACL), - binary_acl = relation(BinaryACL)), + acl = relation(ACL)), extension = validator) mapper(Keyring, self.tbl_keyrings, properties = dict(keyring_name = self.tbl_keyrings.c.name, - keyring_id = self.tbl_keyrings.c.id)) + keyring_id = self.tbl_keyrings.c.id, + acl = relation(ACL, primaryjoin=(self.tbl_keyrings.c.acl_id == self.tbl_acl.c.id)))), mapper(DBChange, self.tbl_changes, properties = dict(change_id = self.tbl_changes.c.id, @@ -2738,11 +2712,6 @@ class DBConn(object): date = self.tbl_changes.c.date, version = self.tbl_changes.c.version)) - mapper(KeyringACLMap, self.tbl_keyring_acl_map, - properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, - keyring = relation(Keyring, backref="keyring_acl_map"), - architecture = relation(Architecture))) - mapper(Maintainer, self.tbl_maintainer, properties = dict(maintainer_id = self.tbl_maintainer.c.id, maintains_sources = relation(DBSource, backref='maintainer', @@ -2752,7 +2721,8 @@ class DBConn(object): extension = validator) mapper(NewComment, self.tbl_new_comments, - properties = dict(comment_id = self.tbl_new_comments.c.id)) + properties = dict(comment_id = self.tbl_new_comments.c.id, + policy_queue = relation(PolicyQueue))) mapper(Override, self.tbl_override, properties = dict(suite_id = self.tbl_override.c.suite, @@ -2802,6 +2772,8 @@ class DBConn(object): properties = dict(section_id = self.tbl_section.c.id, section=self.tbl_section.c.section)) + mapper(SignatureHistory, self.tbl_signature_history) + mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, version = self.tbl_source.c.version, @@ -2821,9 +2793,6 @@ class DBConn(object): collection_class=attribute_mapped_collection('key'))), extension = validator) - mapper(SourceACL, self.tbl_source_acl, - properties = dict(source_acl_id = self.tbl_source_acl.c.id)) - mapper(SrcFormat, self.tbl_src_format, properties = dict(src_format_id = self.tbl_src_format.c.id, format_name = self.tbl_src_format.c.format_name)) @@ -2831,11 +2800,16 @@ class DBConn(object): mapper(Suite, self.tbl_suite, properties = dict(suite_id = self.tbl_suite.c.id, policy_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.policy_queue_id == self.tbl_policy_queue.c.id)), + new_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.new_queue_id == self.tbl_policy_queue.c.id)), copy_queues = relation(BuildQueue, secondary=self.tbl_suite_build_queue_copy), srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats, backref=backref('suites', lazy='dynamic')), - archive = relation(Archive, backref='suites')), + archive = relation(Archive, backref='suites'), + acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set), + components = relation(Component, secondary=self.tbl_component_suite, + order_by=self.tbl_component.c.ordering, + backref=backref('suites'))), extension = validator) mapper(Uid, self.tbl_uid, @@ -2843,11 +2817,6 @@ class DBConn(object): fingerprint = relation(Fingerprint)), extension = validator) - mapper(UploadBlock, self.tbl_upload_blocks, - properties = dict(upload_block_id = self.tbl_upload_blocks.c.id, - fingerprint = relation(Fingerprint, backref="uploadblocks"), - uid = relation(Uid, backref="uploadblocks"))) - mapper(BinContents, self.tbl_bin_contents, properties = dict( binary = relation(DBBinary, @@ -2911,7 +2880,7 @@ class DBConn(object): engine_args['pool_size'] = int(cnf['DB::PoolSize']) if cnf.has_key('DB::MaxOverflow'): engine_args['max_overflow'] = int(cnf['DB::MaxOverflow']) - if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \ + if sa_major_version != '0.5' and cnf.has_key('DB::Unicode') and \ cnf['DB::Unicode'] == 'false': engine_args['use_native_unicode'] = False @@ -2963,5 +2932,3 @@ class DBConn(object): return session __all__.append('DBConn') - -