X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=4abfcc596f934fdefa52b4381d26a9e147bfc4f7;hb=a99beb83431df83aedea355d915d8a776cced0a3;hp=a34e5c75e8ebb57a01e18fbf55c74088741d22d0;hpb=6664882fd3d28f81a412295bbac85c72a510fce3;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index a34e5c75..4abfcc59 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -45,7 +45,7 @@ from tempfile import mkstemp, mkdtemp from inspect import getargspec import sqlalchemy -from sqlalchemy import create_engine, Table, MetaData +from sqlalchemy import create_engine, Table, MetaData, Column, Integer from sqlalchemy.orm import sessionmaker, mapper, relation from sqlalchemy import types as sqltypes @@ -57,30 +57,41 @@ from sqlalchemy.orm.exc import NoResultFound # in the database from config import Config from textutils import fix_maintainer +from dak_exceptions import NoSourceFieldError ################################################################################ # Patch in support for the debversion field type so that it works during # reflection -class DebVersion(sqltypes.Text): - """ - Support the debversion type - """ +try: + # that is for sqlalchemy 0.6 + UserDefinedType = sqltypes.UserDefinedType +except: + # this one for sqlalchemy 0.5 + UserDefinedType = sqltypes.TypeEngine +class DebVersion(UserDefinedType): def get_col_spec(self): return "DEBVERSION" + def bind_processor(self, dialect): + return None + + # ' = None' is needed for sqlalchemy 0.5: + def result_processor(self, dialect, coltype = None): + return None + sa_major_version = sqlalchemy.__version__[0:3] -if sa_major_version == "0.5": +if sa_major_version in ["0.5", "0.6"]: from sqlalchemy.databases import postgres postgres.ischema_names['debversion'] = DebVersion else: - raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py") + raise Exception("dak only ported to SQLA versions 0.5 and 0.6. See daklib/dbconn.py") ################################################################################ -__all__ = ['IntegrityError', 'SQLAlchemyError'] +__all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion'] ################################################################################ @@ -136,8 +147,9 @@ __all__.append('session_wrapper') ################################################################################ class Architecture(object): - def __init__(self, *args, **kwargs): - pass + def __init__(self, arch_string = None, description = None): + self.arch_string = arch_string + self.description = description def __eq__(self, val): if isinstance(val, str): @@ -442,8 +454,8 @@ MINIMAL_APT_CONF=""" Dir { ArchiveDir "%(archivepath)s"; - OverrideDir "/srv/backports-master.debian.org/scripts/override/"; - CacheDir "/srv/backports-master.debian.org/database/"; + OverrideDir "%(overridedir)s"; + CacheDir "%(cachedir)s"; }; Default @@ -505,11 +517,16 @@ class BuildQueue(object): os.write(fl_fd, '%s\n' % n.fullpath) os.close(fl_fd) + cnf = Config() + # Write minimal apt.conf # TODO: Remove hardcoding from template (ac_fd, ac_name) = mkstemp() os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path, - 'filelist': fl_name}) + 'filelist': fl_name, + 'cachedir': cnf["Dir::Cache"], + 'overridedir': cnf["Dir::Override"], + }) os.close(ac_fd) # Run apt-ftparchive generate @@ -539,7 +556,6 @@ class BuildQueue(object): # Sign if necessary if self.signingkey: - cnf = Config() keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"] if cnf.has_key("Dinstall::SigningPubKeyring"): keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"] @@ -611,7 +627,7 @@ class BuildQueue(object): session.commit() for f in os.listdir(self.path): - if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release'): + if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'): continue try: @@ -1211,8 +1227,8 @@ __all__.append('add_poolfile') ################################################################################ class Fingerprint(object): - def __init__(self, *args, **kwargs): - pass + def __init__(self, fingerprint = None): + self.fingerprint = fingerprint def __repr__(self): return '' % self.fingerprint @@ -1313,9 +1329,17 @@ class Keyring(object): esclist[x] = "%c" % (int(esclist[x][2:],16)) return "".join(esclist) - def load_keys(self, keyring): + def parse_address(self, uid): + """parses uid and returns a tuple of real name and email address""" import email.Utils + (name, address) = email.Utils.parseaddr(uid) + name = re.sub(r"\s*[(].*[)]", "", name) + name = self.de_escape_gpg_str(name) + if name == "": + name = uid + return (name, address) + def load_keys(self, keyring): if not self.keyring_id: raise Exception('Must be initialized with database information') @@ -1327,24 +1351,20 @@ class Keyring(object): field = line.split(":") if field[0] == "pub": key = field[4] - (name, addr) = email.Utils.parseaddr(field[9]) - name = re.sub(r"\s*[(].*[)]", "", name) - if name == "" or addr == "" or "@" not in addr: - name = field[9] - addr = "invalid-uid" - name = self.de_escape_gpg_str(name) - self.keys[key] = {"email": addr} - if name != "": + self.keys[key] = {} + (name, addr) = self.parse_address(field[9]) + if "@" in addr: + self.keys[key]["email"] = addr self.keys[key]["name"] = name - self.keys[key]["aliases"] = [name] self.keys[key]["fingerprints"] = [] signingkey = True elif key and field[0] == "sub" and len(field) >= 12: signingkey = ("s" in field[11]) elif key and field[0] == "uid": - (name, addr) = email.Utils.parseaddr(field[9]) - if name and name not in self.keys[key]["aliases"]: - self.keys[key]["aliases"].append(name) + (name, addr) = self.parse_address(field[9]) + if "email" not in self.keys[key] and "@" in addr: + self.keys[key]["email"] = addr + self.keys[key]["name"] = name elif signingkey and field[0] == "fpr": self.keys[key]["fingerprints"].append(field[9]) self.fpr_lookup[field[9]] = key @@ -1392,7 +1412,7 @@ class Keyring(object): byname = {} any_invalid = False for x in self.keys.keys(): - if self.keys[x]["email"] == "invalid-uid": + if "email" not in self.keys[x]: any_invalid = True self.keys[x]["uid"] = format % "invalid-uid" else: @@ -1913,6 +1933,31 @@ def get_policy_queue(queuename, session=None): __all__.append('get_policy_queue') +@session_wrapper +def get_policy_queue_from_path(pathname, session=None): + """ + Returns PolicyQueue object for given C{path name} + + @type queuename: string + @param queuename: The path + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: PolicyQueue + @return: PolicyQueue object for the given queue + """ + + q = session.query(PolicyQueue).filter_by(path=pathname) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_policy_queue_from_path') + ################################################################################ class Priority(object): @@ -2365,7 +2410,7 @@ def add_deb_to_db(u, filename, session=None): bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session) if len(bin_sources) != 1: raise NoSourceFieldError, "Unable to find a unique source id for %s (%s), %s, file %s, type %s, signed by %s" % \ - (bin.package, bin.version, bin.architecture.arch_string, + (bin.package, bin.version, entry["architecture"], filename, bin.binarytype, u.pkg.changes["fingerprint"]) bin.source_id = bin_sources[0].source_id @@ -2454,10 +2499,7 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'), ('Priority', 'priority'), ('NotAutomatic', 'notautomatic'), ('CopyChanges', 'copychanges'), - ('CopyDotDak', 'copydotdak'), - ('CommentsDir', 'commentsdir'), - ('OverrideSuite', 'overridesuite'), - ('ChangelogBase', 'changelogbase')] + ('OverrideSuite', 'overridesuite')] class Suite(object): def __init__(self, *args, **kwargs): @@ -2635,8 +2677,9 @@ __all__.append('get_suite_src_formats') ################################################################################ class Uid(object): - def __init__(self, *args, **kwargs): - pass + def __init__(self, uid = None, name = None): + self.uid = uid + self.name = name def __eq__(self, val): if isinstance(val, str): @@ -2729,35 +2772,28 @@ class DBConn(object): self.__createconn() def __setuptables(self): - tables = ( + tables_with_primary = ( 'architecture', 'archive', 'bin_associations', 'binaries', 'binary_acl', 'binary_acl_map', - 'bin_contents', 'build_queue', - 'build_queue_files', + 'changelogs_text', 'component', 'config', 'changes_pending_binaries', 'changes_pending_files', - 'changes_pending_files_map', 'changes_pending_source', - 'changes_pending_source_files', - 'changes_pool_files', - 'deb_contents', 'dsc_files', 'files', 'fingerprint', 'keyrings', - 'changes', 'keyring_acl_map', 'location', 'maintainer', 'new_comments', - 'override', 'override_type', 'pending_bin_contents', 'policy_queue', @@ -2769,18 +2805,72 @@ class DBConn(object): 'src_format', 'src_uploaders', 'suite', + 'uid', + 'upload_blocks', + # The following tables have primary keys but sqlalchemy + # version 0.5 fails to reflect them correctly with database + # versions before upgrade #41. + #'changes', + #'build_queue_files', + ) + + tables_no_primary = ( + 'bin_contents', + 'changes_pending_files_map', + 'changes_pending_source_files', + 'changes_pool_files', + 'deb_contents', + 'override', 'suite_architectures', 'suite_src_formats', 'suite_build_queue_copy', 'udeb_contents', - 'uid', - 'upload_blocks', + # see the comment above + 'changes', + 'build_queue_files', + ) + + views = ( + 'almost_obsolete_all_associations', + 'almost_obsolete_src_associations', + 'any_associations_source', + 'bin_assoc_by_arch', + 'bin_associations_binaries', + 'binaries_suite_arch', + 'binfiles_suite_component_arch', + 'changelogs', + 'file_arch_suite', + 'newest_all_associations', + 'newest_any_associations', + 'newest_source', + 'newest_src_association', + 'obsolete_all_associations', + 'obsolete_any_associations', + 'obsolete_any_by_all_associations', + 'obsolete_src_associations', + 'source_suite', + 'src_associations_bin', + 'src_associations_src', + 'suite_arch_by_name', ) - for table_name in tables: + # Sqlalchemy version 0.5 fails to reflect the SERIAL type + # correctly and that is why we have to use a workaround. It can + # be removed as soon as we switch to version 0.6. + for table_name in tables_with_primary: + table = Table(table_name, self.db_meta, \ + Column('id', Integer, primary_key = True), \ + autoload=True, useexisting=True) + setattr(self, 'tbl_%s' % table_name, table) + + for table_name in tables_no_primary: table = Table(table_name, self.db_meta, autoload=True) setattr(self, 'tbl_%s' % table_name, table) + for view_name in views: + view = Table(view_name, self.db_meta, autoload=True) + setattr(self, 'view_%s' % view_name, view) + def __setupmappers(self): mapper(Architecture, self.tbl_architecture, properties = dict(arch_id = self.tbl_architecture.c.id))