X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=f7ef853dc6cce6bfe531facd5b298bb8bf5a5337;hb=29586333f60d53b4a1d4e8c3580ae5c3256c8bd9;hp=651b790b194111ae2bd659fee74ce52a82140d1a;hpb=c02076be73a969f103f12c91ef67250d6e4c08d2;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 651b790b..f7ef853d 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -34,16 +34,21 @@ ################################################################################ import os +import re import psycopg2 import traceback +from datetime import datetime from inspect import getargspec -from sqlalchemy import create_engine, Table, MetaData, select +import sqlalchemy +from sqlalchemy import create_engine, Table, MetaData from sqlalchemy.orm import sessionmaker, mapper, relation +from sqlalchemy import types as sqltypes # Don't remove this, we re-export the exceptions to scripts which import us from sqlalchemy.exc import * +from sqlalchemy.orm.exc import NoResultFound # Only import Config until Queue stuff is changed to store its config # in the database @@ -53,29 +58,75 @@ from textutils import fix_maintainer ################################################################################ +# Patch in support for the debversion field type so that it works during +# reflection + +class DebVersion(sqltypes.Text): + def get_col_spec(self): + return "DEBVERSION" + +sa_major_version = sqlalchemy.__version__[0:3] +if sa_major_version == "0.5": + from sqlalchemy.databases import postgres + postgres.ischema_names['debversion'] = DebVersion +else: + raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py") + +################################################################################ + __all__ = ['IntegrityError', 'SQLAlchemyError'] ################################################################################ def session_wrapper(fn): + """ + Wrapper around common ".., session=None):" handling. If the wrapped + function is called without passing 'session', we create a local one + and destroy it when the function ends. + + Also attaches a commit_or_flush method to the session; if we created a + local session, this is a synonym for session.commit(), otherwise it is a + synonym for session.flush(). + """ + def wrapped(*args, **kwargs): private_transaction = False + + # Find the session object session = kwargs.get('session') - # No session specified as last argument or in kwargs, create one. - if session is None and len(args) == len(getargspec(fn)[0]) - 1: - private_transaction = True - kwargs['session'] = DBConn().session() + if session is None: + if len(args) <= len(getargspec(fn)[0]) - 1: + # No session specified as last argument or in kwargs + private_transaction = True + session = kwargs['session'] = DBConn().session() + else: + # Session is last argument in args + session = args[-1] + if session is None: + args = list(args) + session = args[-1] = DBConn().session() + private_transaction = True + + if private_transaction: + session.commit_or_flush = session.commit + else: + session.commit_or_flush = session.flush try: return fn(*args, **kwargs) finally: if private_transaction: # We created a session; close it. - kwargs['session'].close() + session.close() + + wrapped.__doc__ = fn.__doc__ + wrapped.func_name = fn.func_name return wrapped +__all__.append('session_wrapper') + ################################################################################ class Architecture(object): @@ -117,12 +168,10 @@ def get_architecture(architecture, session=None): q = session.query(Architecture).filter_by(arch_string=architecture) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_architecture') @@ -159,14 +208,14 @@ class Archive(object): pass def __repr__(self): - return '' % self.name + return '' % self.archive_name __all__.append('Archive') @session_wrapper def get_archive(archive, session=None): """ - returns database id for given c{archive}. + returns database id for given C{archive}. @type archive: string @param archive: the name of the arhive @@ -183,12 +232,10 @@ def get_archive(archive, session=None): q = session.query(Archive).filter_by(archive_name=archive) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_archive') @@ -205,6 +252,17 @@ __all__.append('BinAssociation') ################################################################################ +class BinContents(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.binary, self.filename) + +__all__.append('BinContents') + +################################################################################ + class DBBinary(object): def __init__(self, *args, **kwargs): pass @@ -226,19 +284,17 @@ def get_suites_binary_in(package, session=None): @return: list of Suite objects for the given package """ - ret = session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all() - - return ret + return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all() __all__.append('get_suites_binary_in') @session_wrapper -def get_binary_from_id(id, session=None): +def get_binary_from_id(binary_id, session=None): """ Returns DBBinary object for given C{id} - @type id: int - @param id: Id of the required binary + @type binary_id: int + @param binary_id: Id of the required binary @type session: Session @param session: Optional SQLA session object (a temporary one will be @@ -248,14 +304,12 @@ def get_binary_from_id(id, session=None): @return: DBBinary object for the given binary (None if not present) """ - q = session.query(DBBinary).filter_by(binary_id=id) - - if q.count() == 0: - ret = None - else: - ret = q.one() + q = session.query(DBBinary).filter_by(binary_id=binary_id) - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_binary_from_id') @@ -313,9 +367,7 @@ def get_binaries_from_source_id(source_id, session=None): @return: list of DBBinary objects for the given name (may be empty) """ - ret = session.query(DBBinary).filter_by(source_id=source_id).all() - - return ret + return session.query(DBBinary).filter_by(source_id=source_id).all() __all__.append('get_binaries_from_source_id') @@ -335,9 +387,7 @@ def get_binary_from_name_suite(package, suitename, session=None): AND su.suite_name=:suitename ORDER BY b.version DESC""" - ret = session.execute(sql, {'package': package, 'suitename': suitename}) - - return ret + return session.execute(sql, {'package': package, 'suitename': suitename}) __all__.append('get_binary_from_name_suite') @@ -354,14 +404,34 @@ def get_binary_components(package, suitename, arch, session=None): vals = {'package': package, 'suitename': suitename, 'arch': arch} - ret = session.execute(query, vals) - - return ret + return session.execute(query, vals) __all__.append('get_binary_components') ################################################################################ +class BinaryACL(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.binary_acl_id + +__all__.append('BinaryACL') + +################################################################################ + +class BinaryACLMap(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.binary_acl_map_id + +__all__.append('BinaryACLMap') + +################################################################################ + class Component(object): def __init__(self, *args, **kwargs): pass @@ -400,12 +470,10 @@ def get_component(component, session=None): q = session.query(Component).filter_by(component_name=component) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_component') @@ -422,15 +490,7 @@ __all__.append('DBConfig') ################################################################################ -class ContentFilename(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.filename - -__all__.append('ContentFilename') - +@session_wrapper def get_or_set_contents_file_id(filename, session=None): """ Returns database id for given filename. @@ -447,26 +507,17 @@ def get_or_set_contents_file_id(filename, session=None): @rtype: int @return: the database id for the given component """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True q = session.query(ContentFilename).filter_by(filename=filename) - if q.count() < 1: + + try: + ret = q.one().cafilename_id + except NoResultFound: cf = ContentFilename() cf.filename = filename session.add(cf) - if privatetrans: - session.commit() - else: - session.flush() + session.commit_or_flush() ret = cf.cafilename_id - else: - ret = q.one().cafilename_id - - if privatetrans: - session.close() return ret @@ -519,9 +570,7 @@ def get_contents(suite, overridetype, section=None, session=None): contents_q += " ORDER BY fn" - ret = session.execute(contents_q, vals) - - return ret + return session.execute(contents_q, vals) __all__.append('get_contents') @@ -536,6 +585,7 @@ class ContentFilepath(object): __all__.append('ContentFilepath') +@session_wrapper def get_or_set_contents_path_id(filepath, session=None): """ Returns database id for given path. @@ -552,26 +602,17 @@ def get_or_set_contents_path_id(filepath, session=None): @rtype: int @return: the database id for the given path """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True q = session.query(ContentFilepath).filter_by(filepath=filepath) - if q.count() < 1: + + try: + ret = q.one().cafilepath_id + except NoResultFound: cf = ContentFilepath() cf.filepath = filepath session.add(cf) - if privatetrans: - session.commit() - else: - session.flush() + session.commit_or_flush() ret = cf.cafilepath_id - else: - ret = q.one().cafilepath_id - - if privatetrans: - session.close() return ret @@ -615,28 +656,14 @@ def insert_content_paths(binary_id, fullpaths, session=None): # Insert paths pathcache = {} for fullpath in fullpaths: - # Get the necessary IDs ... - (path, file) = os.path.split(fullpath) - - filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) - - pathcache[fullpath] = (filepath_id, filename_id) + if fullpath.startswith( './' ): + fullpath = fullpath[2:] - for fullpath, dat in pathcache.items(): - ca = ContentAssociation() - ca.binary_id = binary_id - ca.filepath_id = dat[0] - ca.filename_id = dat[1] - session.add(ca) + session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} ) - # Only commit if we set up the session ourself + session.commit() if privatetrans: - session.commit() session.close() - else: - session.flush() - return True except: @@ -691,9 +718,7 @@ def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None if poolfile_id is not None: q = q.filter_by(poolfile_id=poolfile_id) - ret = q.all() - - return ret + return q.all() __all__.append('get_dscfiles') @@ -747,7 +772,7 @@ def check_poolfile(filename, filesize, md5sum, location_id, session=None): ret = (False, None) else: obj = q.one() - if obj.md5sum != md5sum or obj.filesize != filesize: + if obj.md5sum != md5sum or obj.filesize != int(filesize): ret = (False, obj) if ret is None: @@ -771,12 +796,10 @@ def get_poolfile_by_id(file_id, session=None): q = session.query(PoolFile).filter_by(file_id=file_id) - if q.count() > 0: - ret = q.one() - else: - ret = None - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_poolfile_by_id') @@ -802,9 +825,7 @@ def get_poolfile_by_name(filename, location_id=None, session=None): if location_id is not None: q = q.join(Location).filter_by(location_id=location_id) - ret = q.all() - - return ret + return q.all() __all__.append('get_poolfile_by_name') @@ -823,12 +844,43 @@ def get_poolfile_like_name(filename, session=None): # TODO: There must be a way of properly using bind parameters with %FOO% q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename)) - ret = q.all() - - return ret + return q.all() __all__.append('get_poolfile_like_name') +@session_wrapper +def add_poolfile(filename, datadict, location_id, session=None): + """ + Add a new file to the pool + + @type filename: string + @param filename: filename + + @type datadict: dict + @param datadict: dict with needed data + + @type location_id: int + @param location_id: database id of the location + + @rtype: PoolFile + @return: the PoolFile object created + """ + poolfile = PoolFile() + poolfile.filename = filename + poolfile.filesize = datadict["size"] + poolfile.md5sum = datadict["md5sum"] + poolfile.sha1sum = datadict["sha1sum"] + poolfile.sha256sum = datadict["sha256sum"] + poolfile.location_id = location_id + + session.add(poolfile) + # Flush to get a file id (NB: This is not a commit) + session.flush() + + return poolfile + +__all__.append('add_poolfile') + ################################################################################ class Fingerprint(object): @@ -840,6 +892,34 @@ class Fingerprint(object): __all__.append('Fingerprint') +@session_wrapper +def get_fingerprint(fpr, session=None): + """ + Returns Fingerprint object for given fpr. + + @type fpr: string + @param fpr: The fpr to find / add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). + + @rtype: Fingerprint + @return: the Fingerprint object for the given fpr or None + """ + + q = session.query(Fingerprint).filter_by(fingerprint=fpr) + + try: + ret = q.one() + except NoResultFound: + ret = None + + return ret + +__all__.append('get_fingerprint') + +@session_wrapper def get_or_set_fingerprint(fpr, session=None): """ Returns Fingerprint object for given fpr. @@ -858,26 +938,17 @@ def get_or_set_fingerprint(fpr, session=None): @rtype: Fingerprint @return: the Fingerprint object for the given fpr """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True q = session.query(Fingerprint).filter_by(fingerprint=fpr) - if q.count() < 1: + + try: + ret = q.one() + except NoResultFound: fingerprint = Fingerprint() fingerprint.fingerprint = fpr session.add(fingerprint) - if privatetrans: - session.commit() - else: - session.flush() + session.commit_or_flush() ret = fingerprint - else: - ret = q.one() - - if privatetrans: - session.close() return ret @@ -885,19 +956,139 @@ __all__.append('get_or_set_fingerprint') ################################################################################ +# Helper routine for Keyring class +def get_ldap_name(entry): + name = [] + for k in ["cn", "mn", "sn"]: + ret = entry.get(k) + if ret and ret[0] != "" and ret[0] != "-": + name.append(ret[0]) + return " ".join(name) + +################################################################################ + class Keyring(object): + gpg_invocation = "gpg --no-default-keyring --keyring %s" +\ + " --with-colons --fingerprint --fingerprint" + + keys = {} + fpr_lookup = {} + def __init__(self, *args, **kwargs): pass def __repr__(self): return '' % self.keyring_name + def de_escape_gpg_str(self, txt): + esclist = re.split(r'(\\x..)', txt) + for x in range(1,len(esclist),2): + esclist[x] = "%c" % (int(esclist[x][2:],16)) + return "".join(esclist) + + def load_keys(self, keyring): + import email.Utils + + if not self.keyring_id: + raise Exception('Must be initialized with database information') + + k = os.popen(self.gpg_invocation % keyring, "r") + key = None + signingkey = False + + for line in k.xreadlines(): + field = line.split(":") + if field[0] == "pub": + key = field[4] + (name, addr) = email.Utils.parseaddr(field[9]) + name = re.sub(r"\s*[(].*[)]", "", name) + if name == "" or addr == "" or "@" not in addr: + name = field[9] + addr = "invalid-uid" + name = self.de_escape_gpg_str(name) + self.keys[key] = {"email": addr} + if name != "": + self.keys[key]["name"] = name + self.keys[key]["aliases"] = [name] + self.keys[key]["fingerprints"] = [] + signingkey = True + elif key and field[0] == "sub" and len(field) >= 12: + signingkey = ("s" in field[11]) + elif key and field[0] == "uid": + (name, addr) = email.Utils.parseaddr(field[9]) + if name and name not in self.keys[key]["aliases"]: + self.keys[key]["aliases"].append(name) + elif signingkey and field[0] == "fpr": + self.keys[key]["fingerprints"].append(field[9]) + self.fpr_lookup[field[9]] = key + + def import_users_from_ldap(self, session): + import ldap + cnf = Config() + + LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"] + LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"] + + l = ldap.open(LDAPServer) + l.simple_bind_s("","") + Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, + "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]), + ["uid", "keyfingerprint", "cn", "mn", "sn"]) + + ldap_fin_uid_id = {} + + byuid = {} + byname = {} + + for i in Attrs: + entry = i[1] + uid = entry["uid"][0] + name = get_ldap_name(entry) + fingerprints = entry["keyFingerPrint"] + keyid = None + for f in fingerprints: + key = self.fpr_lookup.get(f, None) + if key not in self.keys: + continue + self.keys[key]["uid"] = uid + + if keyid != None: + continue + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, name) + byname[uid] = (keyid, name) + + return (byname, byuid) + + def generate_users_from_keyring(self, format, session): + byuid = {} + byname = {} + any_invalid = False + for x in self.keys.keys(): + if self.keys[x]["email"] == "invalid-uid": + any_invalid = True + self.keys[x]["uid"] = format % "invalid-uid" + else: + uid = format % self.keys[x]["email"] + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, self.keys[x]["name"]) + byname[uid] = (keyid, self.keys[x]["name"]) + self.keys[x]["uid"] = uid + + if any_invalid: + uid = format % "invalid-uid" + keyid = get_or_set_uid(uid, session).uid_id + byuid[keyid] = (uid, "ungeneratable user id") + byname[uid] = (keyid, "ungeneratable user id") + + return (byname, byuid) + __all__.append('Keyring') -def get_or_set_keyring(keyring, session=None): +@session_wrapper +def get_keyring(keyring, session=None): """ - If C{keyring} does not have an entry in the C{keyrings} table yet, create one - and return the new Keyring + If C{keyring} does not have an entry in the C{keyrings} table yet, return None If C{keyring} already has an entry, simply return the existing Keyring @type keyring: string @@ -905,33 +1096,65 @@ def get_or_set_keyring(keyring, session=None): @rtype: Keyring @return: the Keyring object for this keyring - """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True + + q = session.query(Keyring).filter_by(keyring_name=keyring) try: - obj = session.query(Keyring).filter_by(keyring_name=keyring).first() + return q.one() + except NoResultFound: + return None - if obj is None: - obj = Keyring(keyring_name=keyring) - session.add(obj) - if privatetrans: - session.commit() - else: - session.flush() +__all__.append('get_keyring') - return obj - finally: - if privatetrans: - session.close() +################################################################################ + +class KeyringACLMap(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.keyring_acl_map_id -__all__.append('get_or_set_keyring') +__all__.append('KeyringACLMap') ################################################################################ +class KnownChange(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.changesname + +__all__.append('KnownChange') + +@session_wrapper +def get_knownchange(filename, session=None): + """ + returns knownchange object for given C{filename}. + + @type archive: string + @param archive: the name of the arhive + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Archive + @return: Archive object for the given name (None if not present) + + """ + q = session.query(KnownChange).filter_by(changesname=filename) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_knownchange') + +################################################################################ class Location(object): def __init__(self, *args, **kwargs): pass @@ -968,12 +1191,10 @@ def get_location(location, component=None, archive=None, session=None): if component is not None: q = q.join(Component).filter_by(component_name=component) - if q.count() < 1: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_location') @@ -994,6 +1215,7 @@ class Maintainer(object): __all__.append('Maintainer') +@session_wrapper def get_or_set_maintainer(name, session=None): """ Returns Maintainer object for given maintainer name. @@ -1012,31 +1234,22 @@ def get_or_set_maintainer(name, session=None): @rtype: Maintainer @return: the Maintainer object for the given maintainer """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True q = session.query(Maintainer).filter_by(name=name) - if q.count() < 1: + try: + ret = q.one() + except NoResultFound: maintainer = Maintainer() maintainer.name = name session.add(maintainer) - if privatetrans: - session.commit() - else: - session.flush() + session.commit_or_flush() ret = maintainer - else: - ret = q.one() - - if privatetrans: - session.close() return ret __all__.append('get_or_set_maintainer') +@session_wrapper def get_maintainer(maintainer_id, session=None): """ Return the name of the maintainer behind C{maintainer_id} or None if that @@ -1049,16 +1262,7 @@ def get_maintainer(maintainer_id, session=None): @return: the Maintainer with this C{maintainer_id} """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - - try: - return session.query(Maintainer).get(maintainer_id) - finally: - if privatetrans: - session.close() + return session.query(Maintainer).get(maintainer_id) __all__.append('get_maintainer') @@ -1096,9 +1300,7 @@ def has_new_comment(package, version, session=None): q = q.filter_by(package=package) q = q.filter_by(version=version) - ret = q.count() > 0 - - return ret + return bool(q.count() > 0) __all__.append('has_new_comment') @@ -1130,9 +1332,7 @@ def get_new_comments(package=None, version=None, comment_id=None, session=None): if version is not None: q = q.filter_by(version=version) if comment_id is not None: q = q.filter_by(comment_id=comment_id) - ret = q.all() - - return ret + return q.all() __all__.append('get_new_comments') @@ -1190,9 +1390,7 @@ def get_override(package, suite=None, component=None, overridetype=None, session if not isinstance(overridetype, list): overridetype = [overridetype] q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype)) - ret = q.all() - - return ret + return q.all() __all__.append('get_override') @@ -1226,12 +1424,10 @@ def get_override_type(override_type, session=None): q = session.query(OverrideType).filter_by(overridetype=override_type) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_override_type') @@ -1284,13 +1480,13 @@ def insert_pending_content_paths(package, fullpaths, session=None): # Insert paths pathcache = {} for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) + (path, filename) = os.path.split(fullpath) if path.startswith( "./" ): path = path[2:] filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) + filename_id = get_or_set_contents_file_id(filename, session) pathcache[fullpath] = (filepath_id, filename_id) @@ -1364,12 +1560,10 @@ def get_priority(priority, session=None): q = session.query(Priority).filter_by(priority=priority) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_priority') @@ -1460,6 +1654,7 @@ class Queue(object): # TODO: Move into database as above if conf.FindB("Dinstall::SecurityQueueBuild"): # Copy it since the original won't be readable by www-data + import utils utils.copy(src, dest) else: # Create a symlink to it @@ -1473,23 +1668,27 @@ class Queue(object): session.add(qb) - # If the .orig.tar.gz is in the pool, create a symlink to - # it (if one doesn't already exist) - if changes.orig_tar_id: - # Determine the .orig.tar.gz file name - for dsc_file in changes.dsc_files.keys(): - if dsc_file.endswith(".orig.tar.gz"): - filename = dsc_file - - dest = os.path.join(dest_dir, filename) + # If the .orig tarballs are in the pool, create a symlink to + # them (if one doesn't already exist) + for dsc_file in changes.dsc_files.keys(): + # Skip all files except orig tarballs + from daklib.regexes import re_is_orig_source + if not re_is_orig_source.match(dsc_file): + continue + # Skip orig files not identified in the pool + if not (changes.orig_files.has_key(dsc_file) and + changes.orig_files[dsc_file].has_key("id")): + continue + orig_file_id = changes.orig_files[dsc_file]["id"] + dest = os.path.join(dest_dir, dsc_file) # If it doesn't exist, create a symlink if not os.path.exists(dest): q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", - {'id': changes.orig_tar_id}) + {'id': orig_file_id}) res = q.fetchone() if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % (changes.orig_tar_id) + return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id) src = os.path.join(res[0], res[1]) os.symlink(src, dest) @@ -1519,9 +1718,10 @@ class Queue(object): __all__.append('Queue') @session_wrapper -def get_queue(queuename, session=None): +def get_or_set_queue(queuename, session=None): """ - Returns Queue object for given C{queue name}. + Returns Queue object for given C{queue name}, creating it if it does not + exist. @type queuename: string @param queuename: The name of the queue @@ -1535,14 +1735,19 @@ def get_queue(queuename, session=None): """ q = session.query(Queue).filter_by(queue_name=queuename) - if q.count() == 0: - ret = None - else: + + try: ret = q.one() + except NoResultFound: + queue = Queue() + queue.queue_name = queuename + session.add(queue) + session.commit_or_flush() + ret = queue return ret -__all__.append('get_queue') +__all__.append('get_or_set_queue') ################################################################################ @@ -1580,12 +1785,10 @@ def get_queue_build(filename, suite, session=None): q = session.query(QueueBuild).filter_by(filename=filename) q = q.join(Suite).filter_by(suite_name=suite) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_queue_build') @@ -1629,12 +1832,11 @@ def get_section(section, session=None): """ q = session.query(Section).filter_by(section=section) - if q.count() == 0: - ret = None - else: - ret = q.one() - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_section') @@ -1751,9 +1953,7 @@ def get_suites_source_in(source, session=None): @return: list of Suite objects for the given source """ - ret = session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all() - - return ret + return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all() __all__.append('get_suites_source_in') @@ -1788,9 +1988,7 @@ def get_sources_from_name(source, version=None, dm_upload_allowed=None, session= if dm_upload_allowed is not None: q = q.filter_by(dm_upload_allowed=dm_upload_allowed) - ret = q.all() - - return ret + return q.all() __all__.append('get_sources_from_name') @@ -1817,18 +2015,193 @@ def get_source_in_suite(source, suite, session=None): q = q.join('source').filter_by(source=source) q = q.join('suite').filter_by(suite_name=suite) - if q.count() == 0: - ret = None - else: - # ???: Maybe we should just return the SrcAssociation object instead - ret = q.one().source - - return ret + try: + return q.one().source + except NoResultFound: + return None __all__.append('get_source_in_suite') ################################################################################ +@session_wrapper +def add_dsc_to_db(u, filename, session=None): + entry = u.pkg.files[filename] + source = DBSource() + + source.source = u.pkg.dsc["source"] + source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch + source.maintainer_id = get_or_set_maintainer(u.pkg.dsc["maintainer"], session).maintainer_id + source.changedby_id = get_or_set_maintainer(u.pkg.changes["changed-by"], session).maintainer_id + source.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id + source.install_date = datetime.now().date() + + dsc_component = entry["component"] + dsc_location_id = entry["location id"] + + source.dm_upload_allowed = (u.pkg.dsc.get("dm-upload-allowed", '') == "yes") + + # Set up a new poolfile if necessary + if not entry.has_key("files id") or not entry["files id"]: + filename = entry["pool name"] + filename + poolfile = add_poolfile(filename, entry, dsc_location_id, session) + entry["files id"] = poolfile.file_id + + source.poolfile_id = entry["files id"] + session.add(source) + session.flush() + + for suite_name in u.pkg.changes["distribution"].keys(): + sa = SrcAssociation() + sa.source_id = source.source_id + sa.suite_id = get_suite(suite_name).suite_id + session.add(sa) + + session.flush() + + # Add the source files to the DB (files and dsc_files) + dscfile = DSCFile() + dscfile.source_id = source.source_id + dscfile.poolfile_id = entry["files id"] + session.add(dscfile) + + for dsc_file, dentry in u.pkg.dsc_files.items(): + df = DSCFile() + df.source_id = source.source_id + + # If the .orig tarball is already in the pool, it's + # files id is stored in dsc_files by check_dsc(). + files_id = dentry.get("files id", None) + + # Find the entry in the files hash + # TODO: Bail out here properly + dfentry = None + for f, e in u.pkg.files.items(): + if f == dsc_file: + dfentry = e + break + + if files_id is None: + filename = dfentry["pool name"] + dsc_file + + (found, obj) = check_poolfile(filename, dentry["size"], dentry["md5sum"], dsc_location_id) + # FIXME: needs to check for -1/-2 and or handle exception + if found and obj is not None: + files_id = obj.file_id + + # If still not found, add it + if files_id is None: + # HACK: Force sha1sum etc into dentry + dentry["sha1sum"] = dfentry["sha1sum"] + dentry["sha256sum"] = dfentry["sha256sum"] + poolfile = add_poolfile(filename, dentry, dsc_location_id, session) + files_id = poolfile.file_id + + df.poolfile_id = files_id + session.add(df) + + session.flush() + + # Add the src_uploaders to the DB + uploader_ids = [source.maintainer_id] + if u.pkg.dsc.has_key("uploaders"): + for up in u.pkg.dsc["uploaders"].split(","): + up = up.strip() + uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id) + + added_ids = {} + for up in uploader_ids: + if added_ids.has_key(up): + utils.warn("Already saw uploader %s for source %s" % (up, source.source)) + continue + + added_ids[u]=1 + + su = SrcUploader() + su.maintainer_id = up + su.source_id = source.source_id + session.add(su) + + session.flush() + + return dsc_component, dsc_location_id + +__all__.append('add_dsc_to_db') + +@session_wrapper +def add_deb_to_db(u, filename, session=None): + """ + Contrary to what you might expect, this routine deals with both + debs and udebs. That info is in 'dbtype', whilst 'type' is + 'deb' for both of them + """ + cnf = Config() + entry = u.pkg.files[filename] + + bin = DBBinary() + bin.package = entry["package"] + bin.version = entry["version"] + bin.maintainer_id = get_or_set_maintainer(entry["maintainer"], session).maintainer_id + bin.fingerprint_id = get_or_set_fingerprint(u.pkg.changes["fingerprint"], session).fingerprint_id + bin.arch_id = get_architecture(entry["architecture"], session).arch_id + bin.binarytype = entry["dbtype"] + + # Find poolfile id + filename = entry["pool name"] + filename + fullpath = os.path.join(cnf["Dir::Pool"], filename) + if not entry.get("location id", None): + entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], utils.where_am_i(), session).location_id + + if not entry.get("files id", None): + poolfile = add_poolfile(filename, entry, entry["location id"], session) + entry["files id"] = poolfile.file_id + + bin.poolfile_id = entry["files id"] + + # Find source id + bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session) + if len(bin_sources) != 1: + raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \ + (bin.package, bin.version, bin.architecture.arch_string, + filename, bin.binarytype, u.pkg.changes["fingerprint"]) + + bin.source_id = bin_sources[0].source_id + + # Add and flush object so it has an ID + session.add(bin) + session.flush() + + # Add BinAssociations + for suite_name in u.pkg.changes["distribution"].keys(): + ba = BinAssociation() + ba.binary_id = bin.binary_id + ba.suite_id = get_suite(suite_name).suite_id + session.add(ba) + + session.flush() + + # Deal with contents - disabled for now + #contents = copy_temporary_contents(bin.package, bin.version, bin.architecture.arch_string, os.path.basename(filename), None, session) + #if not contents: + # print "REJECT\nCould not determine contents of package %s" % bin.package + # session.rollback() + # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename) + +__all__.append('add_deb_to_db') + +################################################################################ + +class SourceACL(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.source_acl_id + +__all__.append('SourceACL') + +################################################################################ + class SrcAssociation(object): def __init__(self, *args, **kwargs): pass @@ -1840,6 +2213,17 @@ __all__.append('SrcAssociation') ################################################################################ +class SrcFormat(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.format_name) + +__all__.append('SrcFormat') + +################################################################################ + class SrcUploader(object): def __init__(self, *args, **kwargs): pass @@ -1925,12 +2309,10 @@ def get_suite_architecture(suite, architecture, session=None): q = q.join(Architecture).filter_by(arch_string=architecture) q = q.join(Suite).filter_by(suite_name=suite) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_suite_architecture') @@ -1947,17 +2329,15 @@ def get_suite(suite, session=None): generated if not supplied) @rtype: Suite - @return: Suite object for the requested suite name (None if not presenT) + @return: Suite object for the requested suite name (None if not present) """ q = session.query(Suite).filter_by(suite_name=suite) - if q.count() == 0: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_suite') @@ -2008,14 +2388,48 @@ def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): q = q.order_by('arch_string') - ret = q.all() - - return ret + return q.all() __all__.append('get_suite_architectures') ################################################################################ +class SuiteSrcFormat(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.suite_id, self.src_format_id) + +__all__.append('SuiteSrcFormat') + +@session_wrapper +def get_suite_src_formats(suite, session=None): + """ + Returns list of allowed SrcFormat for C{suite}. + + @type suite: str + @param suite: Suite name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: the list of allowed source formats for I{suite} + """ + + q = session.query(SrcFormat) + q = q.join(SuiteSrcFormat) + q = q.join(Suite).filter_by(suite_name=suite) + q = q.order_by('format_name') + + return q.all() + +__all__.append('get_suite_src_formats') + +################################################################################ + class Uid(object): def __init__(self, *args, **kwargs): pass @@ -2037,6 +2451,7 @@ class Uid(object): __all__.append('Uid') +@session_wrapper def add_database_user(uidname, session=None): """ Adds a database user @@ -2053,19 +2468,12 @@ def add_database_user(uidname, session=None): @return: the uid object for the given uidname """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - session.execute("CREATE USER :uid", {'uid': uidname}) - - if privatetrans: - session.commit() - session.close() + session.commit_or_flush() __all__.append('add_database_user') +@session_wrapper def get_or_set_uid(uidname, session=None): """ Returns uid object for given uidname. @@ -2084,27 +2492,16 @@ def get_or_set_uid(uidname, session=None): @return: the uid object for the given uidname """ - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - q = session.query(Uid).filter_by(uid=uidname) - if q.count() < 1: + try: + ret = q.one() + except NoResultFound: uid = Uid() uid.uid = uidname session.add(uid) - if privatetrans: - session.commit() - else: - session.flush() + session.commit_or_flush() ret = uid - else: - ret = q.one() - - if privatetrans: - session.close() return ret @@ -2115,17 +2512,26 @@ def get_uid_from_fingerprint(fpr, session=None): q = session.query(Uid) q = q.join(Fingerprint).filter_by(fingerprint=fpr) - if q.count() != 1: - ret = None - else: - ret = q.one() - - return ret + try: + return q.one() + except NoResultFound: + return None __all__.append('get_uid_from_fingerprint') ################################################################################ +class UploadBlock(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % (self.source, self.upload_block_id) + +__all__.append('UploadBlock') + +################################################################################ + class DBConn(Singleton): """ database module init. @@ -2144,6 +2550,8 @@ class DBConn(Singleton): self.tbl_archive = Table('archive', self.db_meta, autoload=True) self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) + self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) + self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True) self.tbl_component = Table('component', self.db_meta, autoload=True) self.tbl_config = Table('config', self.db_meta, autoload=True) self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) @@ -2153,6 +2561,8 @@ class DBConn(Singleton): self.tbl_files = Table('files', self.db_meta, autoload=True) self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) + self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True) + self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True) self.tbl_location = Table('location', self.db_meta, autoload=True) self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) @@ -2164,11 +2574,15 @@ class DBConn(Singleton): self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True) self.tbl_section = Table('section', self.db_meta, autoload=True) self.tbl_source = Table('source', self.db_meta, autoload=True) + self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True) + self.tbl_src_format = Table('src_format', self.db_meta, autoload=True) self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True) self.tbl_suite = Table('suite', self.db_meta, autoload=True) self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) + self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) self.tbl_uid = Table('uid', self.db_meta, autoload=True) + self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) def __setupmappers(self): mapper(Architecture, self.tbl_architecture, @@ -2185,6 +2599,7 @@ class DBConn(Singleton): binary_id = self.tbl_bin_associations.c.bin, binary = relation(DBBinary))) + mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, package = self.tbl_binaries.c.package, @@ -2204,6 +2619,14 @@ class DBConn(Singleton): binassociations = relation(BinAssociation, primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin)))) + mapper(BinaryACL, self.tbl_binary_acl, + properties = dict(binary_acl_id = self.tbl_binary_acl.c.id)) + + mapper(BinaryACLMap, self.tbl_binary_acl_map, + properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id, + fingerprint = relation(Fingerprint, backref="binary_acl_map"), + architecture = relation(Architecture))) + mapper(Component, self.tbl_component, properties = dict(component_id = self.tbl_component.c.id, component_name = self.tbl_component.c.name)) @@ -2211,24 +2634,6 @@ class DBConn(Singleton): mapper(DBConfig, self.tbl_config, properties = dict(config_id = self.tbl_config.c.id)) - mapper(ContentAssociation, self.tbl_content_associations, - properties = dict(ca_id = self.tbl_content_associations.c.id, - filename_id = self.tbl_content_associations.c.filename, - filename = relation(ContentFilename), - filepath_id = self.tbl_content_associations.c.filepath, - filepath = relation(ContentFilepath), - binary_id = self.tbl_content_associations.c.binary_pkg, - binary = relation(DBBinary))) - - - mapper(ContentFilename, self.tbl_content_file_names, - properties = dict(cafilename_id = self.tbl_content_file_names.c.id, - filename = self.tbl_content_file_names.c.file)) - - mapper(ContentFilepath, self.tbl_content_file_paths, - properties = dict(cafilepath_id = self.tbl_content_file_paths.c.id, - filepath = self.tbl_content_file_paths.c.path)) - mapper(DSCFile, self.tbl_dsc_files, properties = dict(dscfile_id = self.tbl_dsc_files.c.id, source_id = self.tbl_dsc_files.c.source, @@ -2247,12 +2652,22 @@ class DBConn(Singleton): uid_id = self.tbl_fingerprint.c.uid, uid = relation(Uid), keyring_id = self.tbl_fingerprint.c.keyring, - keyring = relation(Keyring))) + keyring = relation(Keyring), + source_acl = relation(SourceACL), + binary_acl = relation(BinaryACL))) mapper(Keyring, self.tbl_keyrings, properties = dict(keyring_name = self.tbl_keyrings.c.name, keyring_id = self.tbl_keyrings.c.id)) + mapper(KnownChange, self.tbl_known_changes, + properties = dict(known_change_id = self.tbl_known_changes.c.id)) + + mapper(KeyringACLMap, self.tbl_keyring_acl_map, + properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, + keyring = relation(Keyring, backref="keyring_acl_map"), + architecture = relation(Architecture))) + mapper(Location, self.tbl_location, properties = dict(location_id = self.tbl_location.c.id, component_id = self.tbl_location.c.component, @@ -2283,13 +2698,6 @@ class DBConn(Singleton): properties = dict(overridetype = self.tbl_override_type.c.type, overridetype_id = self.tbl_override_type.c.id)) - mapper(PendingContentAssociation, self.tbl_pending_content_associations, - properties = dict(pca_id = self.tbl_pending_content_associations.c.id, - filepath_id = self.tbl_pending_content_associations.c.filepath, - filepath = relation(ContentFilepath), - filename_id = self.tbl_pending_content_associations.c.filename, - filename = relation(ContentFilename))) - mapper(Priority, self.tbl_priority, properties = dict(priority_id = self.tbl_priority.c.id)) @@ -2320,7 +2728,11 @@ class DBConn(Singleton): srcfiles = relation(DSCFile, primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)), srcassociations = relation(SrcAssociation, - primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)))) + primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)), + srcuploaders = relation(SrcUploader))) + + mapper(SourceACL, self.tbl_source_acl, + properties = dict(source_acl_id = self.tbl_source_acl.c.id)) mapper(SrcAssociation, self.tbl_src_associations, properties = dict(sa_id = self.tbl_src_associations.c.id, @@ -2329,6 +2741,10 @@ class DBConn(Singleton): source_id = self.tbl_src_associations.c.source, source = relation(DBSource))) + mapper(SrcFormat, self.tbl_src_format, + properties = dict(src_format_id = self.tbl_src_format.c.id, + format_name = self.tbl_src_format.c.format_name)) + mapper(SrcUploader, self.tbl_src_uploaders, properties = dict(uploader_id = self.tbl_src_uploaders.c.id, source_id = self.tbl_src_uploaders.c.source, @@ -2339,7 +2755,8 @@ class DBConn(Singleton): primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id)))) mapper(Suite, self.tbl_suite, - properties = dict(suite_id = self.tbl_suite.c.id)) + properties = dict(suite_id = self.tbl_suite.c.id, + policy_queue = relation(Queue))) mapper(SuiteArchitecture, self.tbl_suite_architectures, properties = dict(suite_id = self.tbl_suite_architectures.c.suite, @@ -2347,10 +2764,21 @@ class DBConn(Singleton): arch_id = self.tbl_suite_architectures.c.architecture, architecture = relation(Architecture))) + mapper(SuiteSrcFormat, self.tbl_suite_src_formats, + properties = dict(suite_id = self.tbl_suite_src_formats.c.suite, + suite = relation(Suite, backref='suitesrcformats'), + src_format_id = self.tbl_suite_src_formats.c.src_format, + src_format = relation(SrcFormat))) + mapper(Uid, self.tbl_uid, properties = dict(uid_id = self.tbl_uid.c.id, fingerprint = relation(Fingerprint))) + mapper(UploadBlock, self.tbl_upload_blocks, + properties = dict(upload_block_id = self.tbl_upload_blocks.c.id, + fingerprint = relation(Fingerprint, backref="uploadblocks"), + uid = relation(Uid, backref="uploadblocks"))) + ## Connection functions def __createconn(self): from config import Config