X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=9b37af700f631b08fb7990dd4434d440cd6d8d54;hb=63a936065dc2979df325eb34a205c3c97e0cd4ce;hp=48953f60fce3aa0040da1e1f5acddaa04439b3ff;hpb=760a440bb41fcf153c0c0a80aab6e0b71fab03b7;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 48953f60..9b37af70 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -34,12 +34,13 @@ ################################################################################ import os +import re import psycopg2 import traceback from inspect import getargspec -from sqlalchemy import create_engine, Table, MetaData, select +from sqlalchemy import create_engine, Table, MetaData from sqlalchemy.orm import sessionmaker, mapper, relation # Don't remove this, we re-export the exceptions to scripts which import us @@ -73,9 +74,9 @@ def session_wrapper(fn): private_transaction = False # Find the session object - try: - session = kwargs['session'] - except KeyError: + session = kwargs.get('session') + + if session is None: if len(args) <= len(getargspec(fn)[0]) - 1: # No session specified as last argument or in kwargs private_transaction = True @@ -83,6 +84,10 @@ def session_wrapper(fn): else: # Session is last argument in args session = args[-1] + if session is None: + args = list(args) + session = args[-1] = DBConn().session() + private_transaction = True if private_transaction: session.commit_or_flush = session.commit @@ -373,6 +378,28 @@ __all__.append('get_binary_components') ################################################################################ +class BinaryACL(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.binary_acl_id + +__all__.append('BinaryACL') + +################################################################################ + +class BinaryACLMap(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.binary_acl_map_id + +__all__.append('BinaryACLMap') + +################################################################################ + class Component(object): def __init__(self, *args, **kwargs): pass @@ -823,6 +850,33 @@ class Fingerprint(object): __all__.append('Fingerprint') +@session_wrapper +def get_fingerprint(fpr, session=None): + """ + Returns Fingerprint object for given fpr. + + @type fpr: string + @param fpr: The fpr to find / add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). + + @rtype: Fingerprint + @return: the Fingerprint object for the given fpr or None + """ + + q = session.query(Fingerprint).filter_by(fingerprint=fpr) + + try: + ret = q.one() + except NoResultFound: + ret = None + + return ret + +__all__.append('get_fingerprint') + @session_wrapper def get_or_set_fingerprint(fpr, session=None): """ @@ -860,20 +914,139 @@ __all__.append('get_or_set_fingerprint') ################################################################################ +# Helper routine for Keyring class +def get_ldap_name(entry): + name = [] + for k in ["cn", "mn", "sn"]: + ret = entry.get(k) + if ret and ret[0] != "" and ret[0] != "-": + name.append(ret[0]) + return " ".join(name) + +################################################################################ + class Keyring(object): + gpg_invocation = "gpg --no-default-keyring --keyring %s" +\ + " --with-colons --fingerprint --fingerprint" + + keys = {} + fpr_lookup = {} + def __init__(self, *args, **kwargs): pass def __repr__(self): return '' % self.keyring_name + def de_escape_gpg_str(self, str): + esclist = re.split(r'(\\x..)', str) + for x in range(1,len(esclist),2): + esclist[x] = "%c" % (int(esclist[x][2:],16)) + return "".join(esclist) + + def load_keys(self, keyring): + import email.Utils + + if not self.keyring_id: + raise Exception('Must be initialized with database information') + + k = os.popen(self.gpg_invocation % keyring, "r") + key = None + signingkey = False + + for line in k.xreadlines(): + field = line.split(":") + if field[0] == "pub": + key = field[4] + (name, addr) = email.Utils.parseaddr(field[9]) + name = re.sub(r"\s*[(].*[)]", "", name) + if name == "" or addr == "" or "@" not in addr: + name = field[9] + addr = "invalid-uid" + name = self.de_escape_gpg_str(name) + self.keys[key] = {"email": addr} + if name != "": + self.keys[key]["name"] = name + self.keys[key]["aliases"] = [name] + self.keys[key]["fingerprints"] = [] + signingkey = True + elif key and field[0] == "sub" and len(field) >= 12: + signingkey = ("s" in field[11]) + elif key and field[0] == "uid": + (name, addr) = email.Utils.parseaddr(field[9]) + if name and name not in self.keys[key]["aliases"]: + self.keys[key]["aliases"].append(name) + elif signingkey and field[0] == "fpr": + self.keys[key]["fingerprints"].append(field[9]) + self.fpr_lookup[field[9]] = key + + def import_users_from_ldap(self, session): + import ldap + cnf = Config() + + LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"] + LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"] + + l = ldap.open(LDAPServer) + l.simple_bind_s("","") + Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, + "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]), + ["uid", "keyfingerprint", "cn", "mn", "sn"]) + + ldap_fin_uid_id = {} + + byuid = {} + byname = {} + + for i in Attrs: + entry = i[1] + uid = entry["uid"][0] + name = get_ldap_name(entry) + fingerprints = entry["keyFingerPrint"] + keyid = None + for f in fingerprints: + key = self.fpr_lookup.get(f, None) + if key not in self.keys: + continue + self.keys[key]["uid"] = uid + + if keyid != None: + continue + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, name) + byname[uid] = (keyid, name) + + return (byname, byuid) + + def generate_users_from_keyring(self, format, session): + byuid = {} + byname = {} + any_invalid = False + for x in self.keys.keys(): + if self.keys[x]["email"] == "invalid-uid": + any_invalid = True + self.keys[x]["uid"] = format % "invalid-uid" + else: + uid = format % self.keys[x]["email"] + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, self.keys[x]["name"]) + byname[uid] = (keyid, self.keys[x]["name"]) + self.keys[x]["uid"] = uid + + if any_invalid: + uid = format % "invalid-uid" + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, "ungeneratable user id") + byname[uid] = (keyid, "ungeneratable user id") + + return (byname, byuid) + __all__.append('Keyring') @session_wrapper -def get_or_set_keyring(keyring, session=None): +def get_keyring(keyring, session=None): """ - If C{keyring} does not have an entry in the C{keyrings} table yet, create one - and return the new Keyring + If C{keyring} does not have an entry in the C{keyrings} table yet, return None If C{keyring} already has an entry, simply return the existing Keyring @type keyring: string @@ -888,12 +1061,20 @@ def get_or_set_keyring(keyring, session=None): try: return q.one() except NoResultFound: - obj = Keyring(keyring_name=keyring) - session.add(obj) - session.commit_or_flush() - return obj + return None + +__all__.append('get_keyring') + +################################################################################ + +class KeyringACLMap(object): + def __init__(self, *args, **kwargs): + pass -__all__.append('get_or_set_keyring') + def __repr__(self): + return '' % self.keyring_acl_map_id + +__all__.append('KeyringACLMap') ################################################################################ @@ -1410,23 +1591,27 @@ class Queue(object): session.add(qb) - # If the .orig.tar.gz is in the pool, create a symlink to - # it (if one doesn't already exist) - if changes.orig_tar_id: - # Determine the .orig.tar.gz file name - for dsc_file in changes.dsc_files.keys(): - if dsc_file.endswith(".orig.tar.gz"): - filename = dsc_file - - dest = os.path.join(dest_dir, filename) + # If the .orig tarballs are in the pool, create a symlink to + # them (if one doesn't already exist) + for dsc_file in changes.dsc_files.keys(): + # Skip all files except orig tarballs + from daklib.regexes import re_is_orig_source + if not re_is_orig_source.match(dsc_file): + continue + # Skip orig files not identified in the pool + if not (changes.orig_files.has_key(dsc_file) and + changes.orig_files[dsc_file].has_key("id")): + continue + orig_file_id = changes.orig_files[dsc_file]["id"] + dest = os.path.join(dest_dir, dsc_file) # If it doesn't exist, create a symlink if not os.path.exists(dest): q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", - {'id': changes.orig_tar_id}) + {'id': orig_file_id}) res = q.fetchone() if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % (changes.orig_tar_id) + return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id) src = os.path.join(res[0], res[1]) os.symlink(src, dest) @@ -1456,9 +1641,10 @@ class Queue(object): __all__.append('Queue') @session_wrapper -def get_queue(queuename, session=None): +def get_or_set_queue(queuename, session=None): """ - Returns Queue object for given C{queue name}. + Returns Queue object for given C{queue name}, creating it if it does not + exist. @type queuename: string @param queuename: The name of the queue @@ -1474,11 +1660,17 @@ def get_queue(queuename, session=None): q = session.query(Queue).filter_by(queue_name=queuename) try: - return q.one() + ret = q.one() except NoResultFound: - return None + queue = Queue() + queue.queue_name = queuename + session.add(queue) + session.commit_or_flush() + ret = queue + + return ret -__all__.append('get_queue') +__all__.append('get_or_set_queue') ################################################################################ @@ -1755,6 +1947,17 @@ __all__.append('get_source_in_suite') ################################################################################ +class SourceACL(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.source_acl_id + +__all__.append('SourceACL') + +################################################################################ + class SrcAssociation(object): def __init__(self, *args, **kwargs): pass @@ -1766,6 +1969,17 @@ __all__.append('SrcAssociation') ################################################################################ +class SrcFormat(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.format_name) + +__all__.append('SrcFormat') + +################################################################################ + class SrcUploader(object): def __init__(self, *args, **kwargs): pass @@ -1936,6 +2150,42 @@ __all__.append('get_suite_architectures') ################################################################################ +class SuiteSrcFormat(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.suite_id, self.src_format_id) + +__all__.append('SuiteSrcFormat') + +@session_wrapper +def get_suite_src_formats(suite, session=None): + """ + Returns list of allowed SrcFormat for C{suite}. + + @type suite: str + @param suite: Suite name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: the list of allowed source formats for I{suite} + """ + + q = session.query(SrcFormat) + q = q.join(SuiteSrcFormat) + q = q.join(Suite).filter_by(suite_name=suite) + q = q.order_by('format_name') + + return q.all() + +__all__.append('get_suite_src_formats') + +################################################################################ + class Uid(object): def __init__(self, *args, **kwargs): pass @@ -2027,6 +2277,17 @@ __all__.append('get_uid_from_fingerprint') ################################################################################ +class UploadBlock(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.source, self.upload_block_id) + +__all__.append('UploadBlock') + +################################################################################ + class DBConn(Singleton): """ database module init. @@ -2045,6 +2306,8 @@ class DBConn(Singleton): self.tbl_archive = Table('archive', self.db_meta, autoload=True) self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) + self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) + self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True) self.tbl_component = Table('component', self.db_meta, autoload=True) self.tbl_config = Table('config', self.db_meta, autoload=True) self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) @@ -2054,6 +2317,7 @@ class DBConn(Singleton): self.tbl_files = Table('files', self.db_meta, autoload=True) self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) + self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True) self.tbl_location = Table('location', self.db_meta, autoload=True) self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) @@ -2065,11 +2329,15 @@ class DBConn(Singleton): self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True) self.tbl_section = Table('section', self.db_meta, autoload=True) self.tbl_source = Table('source', self.db_meta, autoload=True) + self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True) + self.tbl_src_format = Table('src_format', self.db_meta, autoload=True) self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True) self.tbl_suite = Table('suite', self.db_meta, autoload=True) self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) + self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) self.tbl_uid = Table('uid', self.db_meta, autoload=True) + self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) def __setupmappers(self): mapper(Architecture, self.tbl_architecture, @@ -2105,6 +2373,14 @@ class DBConn(Singleton): binassociations = relation(BinAssociation, primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin)))) + mapper(BinaryACL, self.tbl_binary_acl, + properties = dict(binary_acl_id = self.tbl_binary_acl.c.id)) + + mapper(BinaryACLMap, self.tbl_binary_acl_map, + properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id, + fingerprint = relation(Fingerprint, backref="binary_acl_map"), + architecture = relation(Architecture))) + mapper(Component, self.tbl_component, properties = dict(component_id = self.tbl_component.c.id, component_name = self.tbl_component.c.name)) @@ -2148,12 +2424,19 @@ class DBConn(Singleton): uid_id = self.tbl_fingerprint.c.uid, uid = relation(Uid), keyring_id = self.tbl_fingerprint.c.keyring, - keyring = relation(Keyring))) + keyring = relation(Keyring), + source_acl = relation(SourceACL), + binary_acl = relation(BinaryACL))) mapper(Keyring, self.tbl_keyrings, properties = dict(keyring_name = self.tbl_keyrings.c.name, keyring_id = self.tbl_keyrings.c.id)) + mapper(KeyringACLMap, self.tbl_keyring_acl_map, + properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, + keyring = relation(Keyring, backref="keyring_acl_map"), + architecture = relation(Architecture))) + mapper(Location, self.tbl_location, properties = dict(location_id = self.tbl_location.c.id, component_id = self.tbl_location.c.component, @@ -2221,7 +2504,11 @@ class DBConn(Singleton): srcfiles = relation(DSCFile, primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)), srcassociations = relation(SrcAssociation, - primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)))) + primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)), + srcuploaders = relation(SrcUploader))) + + mapper(SourceACL, self.tbl_source_acl, + properties = dict(source_acl_id = self.tbl_source_acl.c.id)) mapper(SrcAssociation, self.tbl_src_associations, properties = dict(sa_id = self.tbl_src_associations.c.id, @@ -2230,6 +2517,10 @@ class DBConn(Singleton): source_id = self.tbl_src_associations.c.source, source = relation(DBSource))) + mapper(SrcFormat, self.tbl_src_format, + properties = dict(src_format_id = self.tbl_src_format.c.id, + format_name = self.tbl_src_format.c.format_name)) + mapper(SrcUploader, self.tbl_src_uploaders, properties = dict(uploader_id = self.tbl_src_uploaders.c.id, source_id = self.tbl_src_uploaders.c.source, @@ -2248,10 +2539,21 @@ class DBConn(Singleton): arch_id = self.tbl_suite_architectures.c.architecture, architecture = relation(Architecture))) + mapper(SuiteSrcFormat, self.tbl_suite_src_formats, + properties = dict(suite_id = self.tbl_suite_src_formats.c.suite, + suite = relation(Suite, backref='suitesrcformats'), + src_format_id = self.tbl_suite_src_formats.c.src_format, + src_format = relation(SrcFormat))) + mapper(Uid, self.tbl_uid, properties = dict(uid_id = self.tbl_uid.c.id, fingerprint = relation(Fingerprint))) + mapper(UploadBlock, self.tbl_upload_blocks, + properties = dict(upload_block_id = self.tbl_upload_blocks.c.id, + fingerprint = relation(Fingerprint, backref="uploadblocks"), + uid = relation(Uid, backref="uploadblocks"))) + ## Connection functions def __createconn(self): from config import Config