X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=58ad0412232a78f18c01012c1ab5b9b5ba305cb5;hb=391f5ec09a119131dc846b796ca791f4cecc69e4;hp=45e000d395fbf104a8331004e7a95451114ba1f6;hpb=df610ec6b921b869c4e4a25629a0aff3f13a03ae;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 45e000d3..58ad0412 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -5,7 +5,7 @@ @contact: Debian FTPMaster @copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup @copyright: 2008-2009 Mark Hymers -@copyright: 2009 Joerg Jaspert +@copyright: 2009, 2010 Joerg Jaspert @copyright: 2009 Mike O'Connor @license: GNU General Public License version 2 or later """ @@ -33,17 +33,37 @@ ################################################################################ +import apt_pkg +import daklib.daksubprocess import os +from os.path import normpath import re import psycopg2 +import subprocess import traceback +try: + # python >= 2.6 + import json +except: + # python <= 2.5 + import simplejson as json + +from datetime import datetime, timedelta +from errno import ENOENT +from tempfile import mkstemp, mkdtemp +from tarfile import TarFile + from inspect import getargspec import sqlalchemy -from sqlalchemy import create_engine, Table, MetaData -from sqlalchemy.orm import sessionmaker, mapper, relation +from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \ + Text, ForeignKey +from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \ + backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers from sqlalchemy import types as sqltypes +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.ext.associationproxy import association_proxy # Don't remove this, we re-export the exceptions to scripts which import us from sqlalchemy.exc import * @@ -52,28 +72,52 @@ from sqlalchemy.orm.exc import NoResultFound # Only import Config until Queue stuff is changed to store its config # in the database from config import Config -from singleton import Singleton from textutils import fix_maintainer +from dak_exceptions import DBUpdateError, NoSourceFieldError, FileExistsError + +# suppress some deprecation warnings in squeeze related to sqlalchemy +import warnings +warnings.filterwarnings('ignore', \ + "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \ + SADeprecationWarning) +warnings.filterwarnings('ignore', \ + "Predicate of partial index .* ignored during reflection", \ + SAWarning) + ################################################################################ # Patch in support for the debversion field type so that it works during # reflection -class DebVersion(sqltypes.Text): +try: + # that is for sqlalchemy 0.6 + UserDefinedType = sqltypes.UserDefinedType +except: + # this one for sqlalchemy 0.5 + UserDefinedType = sqltypes.TypeEngine + +class DebVersion(UserDefinedType): def get_col_spec(self): return "DEBVERSION" + def bind_processor(self, dialect): + return None + + # ' = None' is needed for sqlalchemy 0.5: + def result_processor(self, dialect, coltype = None): + return None + sa_major_version = sqlalchemy.__version__[0:3] -if sa_major_version == "0.5": - from sqlalchemy.databases import postgres - postgres.ischema_names['debversion'] = DebVersion +if sa_major_version in ["0.5", "0.6", "0.7", "0.8", "0.9", "1.0"]: + from sqlalchemy.databases import postgres + postgres.ischema_names['debversion'] = DebVersion else: - raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py") + raise Exception("dak only ported to SQLA versions 0.5 to 1.0 (%s installed). See daklib/dbconn.py" % sa_major_version) ################################################################################ -__all__ = ['IntegrityError', 'SQLAlchemyError'] +__all__ = ['IntegrityError', 'SQLAlchemyError', 'DebVersion'] ################################################################################ @@ -124,11 +168,224 @@ def session_wrapper(fn): return wrapped +__all__.append('session_wrapper') + ################################################################################ -class Architecture(object): - def __init__(self, *args, **kwargs): - pass +class ORMObject(object): + """ + ORMObject is a base class for all ORM classes mapped by SQLalchemy. All + derived classes must implement the properties() method. + """ + + def properties(self): + ''' + This method should be implemented by all derived classes and returns a + list of the important properties. The properties 'created' and + 'modified' will be added automatically. A suffix '_count' should be + added to properties that are lists or query objects. The most important + property name should be returned as the first element in the list + because it is used by repr(). + ''' + return [] + + def json(self): + ''' + Returns a JSON representation of the object based on the properties + returned from the properties() method. + ''' + data = {} + # add created and modified + all_properties = self.properties() + ['created', 'modified'] + for property in all_properties: + # check for list or query + if property[-6:] == '_count': + real_property = property[:-6] + if not hasattr(self, real_property): + continue + value = getattr(self, real_property) + if hasattr(value, '__len__'): + # list + value = len(value) + elif hasattr(value, 'count'): + # query (but not during validation) + if self.in_validation: + continue + value = value.count() + else: + raise KeyError('Do not understand property %s.' % property) + else: + if not hasattr(self, property): + continue + # plain object + value = getattr(self, property) + if value is None: + # skip None + continue + elif isinstance(value, ORMObject): + # use repr() for ORMObject types + value = repr(value) + else: + # we want a string for all other types because json cannot + # encode everything + value = str(value) + data[property] = value + return json.dumps(data) + + def classname(self): + ''' + Returns the name of the class. + ''' + return type(self).__name__ + + def __repr__(self): + ''' + Returns a short string representation of the object using the first + element from the properties() method. + ''' + primary_property = self.properties()[0] + value = getattr(self, primary_property) + return '<%s %s>' % (self.classname(), str(value)) + + def __str__(self): + ''' + Returns a human readable form of the object using the properties() + method. + ''' + return '<%s %s>' % (self.classname(), self.json()) + + def not_null_constraints(self): + ''' + Returns a list of properties that must be not NULL. Derived classes + should override this method if needed. + ''' + return [] + + validation_message = \ + "Validation failed because property '%s' must not be empty in object\n%s" + + in_validation = False + + def validate(self): + ''' + This function validates the not NULL constraints as returned by + not_null_constraints(). It raises the DBUpdateError exception if + validation fails. + ''' + for property in self.not_null_constraints(): + # TODO: It is a bit awkward that the mapper configuration allow + # directly setting the numeric _id columns. We should get rid of it + # in the long run. + if hasattr(self, property + '_id') and \ + getattr(self, property + '_id') is not None: + continue + if not hasattr(self, property) or getattr(self, property) is None: + # str() might lead to races due to a 2nd flush + self.in_validation = True + message = self.validation_message % (property, str(self)) + self.in_validation = False + raise DBUpdateError(message) + + @classmethod + @session_wrapper + def get(cls, primary_key, session = None): + ''' + This is a support function that allows getting an object by its primary + key. + + Architecture.get(3[, session]) + + instead of the more verbose + + session.query(Architecture).get(3) + ''' + return session.query(cls).get(primary_key) + + def session(self, replace = False): + ''' + Returns the current session that is associated with the object. May + return None is object is in detached state. + ''' + + return object_session(self) + + def clone(self, session = None): + """ + Clones the current object in a new session and returns the new clone. A + fresh session is created if the optional session parameter is not + provided. The function will fail if a session is provided and has + unflushed changes. + + RATIONALE: SQLAlchemy's session is not thread safe. This method clones + an existing object to allow several threads to work with their own + instances of an ORMObject. + + WARNING: Only persistent (committed) objects can be cloned. Changes + made to the original object that are not committed yet will get lost. + The session of the new object will always be rolled back to avoid + resource leaks. + """ + + if self.session() is None: + raise RuntimeError( \ + 'Method clone() failed for detached object:\n%s' % self) + self.session().flush() + mapper = object_mapper(self) + primary_key = mapper.primary_key_from_instance(self) + object_class = self.__class__ + if session is None: + session = DBConn().session() + elif len(session.new) + len(session.dirty) + len(session.deleted) > 0: + raise RuntimeError( \ + 'Method clone() failed due to unflushed changes in session.') + new_object = session.query(object_class).get(primary_key) + session.rollback() + if new_object is None: + raise RuntimeError( \ + 'Method clone() failed for non-persistent object:\n%s' % self) + return new_object + +__all__.append('ORMObject') + +################################################################################ + +class Validator(MapperExtension): + ''' + This class calls the validate() method for each instance for the + 'before_update' and 'before_insert' events. A global object validator is + used for configuring the individual mappers. + ''' + + def before_update(self, mapper, connection, instance): + instance.validate() + return EXT_CONTINUE + + def before_insert(self, mapper, connection, instance): + instance.validate() + return EXT_CONTINUE + +validator = Validator() + +################################################################################ + +class ACL(ORMObject): + def __repr__(self): + return "".format(self.name) + +__all__.append('ACL') + +class ACLPerSource(ORMObject): + def __repr__(self): + return "".format(self.acl.name, self.fingerprint.fingerprint, self.source, self.reason) + +__all__.append('ACLPerSource') + +################################################################################ + +class Architecture(ORMObject): + def __init__(self, arch_string = None, description = None): + self.arch_string = arch_string + self.description = description def __eq__(self, val): if isinstance(val, str): @@ -142,8 +399,11 @@ class Architecture(object): # This signals to use the normal comparison operator return NotImplemented - def __repr__(self): - return '' % self.arch_string + def properties(self): + return ['arch_string', 'arch_id', 'suites_count'] + + def not_null_constraints(self): + return ['arch_string'] __all__.append('Architecture') @@ -172,32 +432,6 @@ def get_architecture(architecture, session=None): __all__.append('get_architecture') -@session_wrapper -def get_architecture_suites(architecture, session=None): - """ - Returns list of Suite objects for given C{architecture} name - - @type source: str - @param source: Architecture name to search for - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of Suite objects for the given name (may be empty) - """ - - q = session.query(Suite) - q = q.join(SuiteArchitecture) - q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name') - - ret = q.all() - - return ret - -__all__.append('get_architecture_suites') - ################################################################################ class Archive(object): @@ -238,34 +472,111 @@ __all__.append('get_archive') ################################################################################ -class BinAssociation(object): - def __init__(self, *args, **kwargs): - pass +class ArchiveFile(object): + def __init__(self, archive=None, component=None, file=None): + self.archive = archive + self.component = component + self.file = file + @property + def path(self): + return os.path.join(self.archive.path, 'pool', self.component.component_name, self.file.filename) - def __repr__(self): - return '' % (self.ba_id, self.binary, self.suite) - -__all__.append('BinAssociation') +__all__.append('ArchiveFile') ################################################################################ -class BinContents(object): - def __init__(self, *args, **kwargs): - pass +class BinContents(ORMObject): + def __init__(self, file = None, binary = None): + self.file = file + self.binary = binary - def __repr__(self): - return '' % (self.binary, self.filename) + def properties(self): + return ['file', 'binary'] __all__.append('BinContents') ################################################################################ -class DBBinary(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.package, self.version, self.architecture) +class DBBinary(ORMObject): + def __init__(self, package = None, source = None, version = None, \ + maintainer = None, architecture = None, poolfile = None, \ + binarytype = 'deb', fingerprint=None): + self.package = package + self.source = source + self.version = version + self.maintainer = maintainer + self.architecture = architecture + self.poolfile = poolfile + self.binarytype = binarytype + self.fingerprint = fingerprint + + @property + def pkid(self): + return self.binary_id + + def properties(self): + return ['package', 'version', 'maintainer', 'source', 'architecture', \ + 'poolfile', 'binarytype', 'fingerprint', 'install_date', \ + 'suites_count', 'binary_id', 'contents_count', 'extra_sources'] + + def not_null_constraints(self): + return ['package', 'version', 'maintainer', 'source', 'poolfile', \ + 'binarytype'] + + metadata = association_proxy('key', 'value') + + def scan_contents(self): + ''' + Yields the contents of the package. Only regular files are yielded and + the path names are normalized after converting them from either utf-8 + or iso8859-1 encoding. It yields the string ' ' if the + package does not contain any regular file. + ''' + fullpath = self.poolfile.fullpath + dpkg_cmd = ('dpkg-deb', '--fsys-tarfile', fullpath) + dpkg = daklib.daksubprocess.Popen(dpkg_cmd, stdout=subprocess.PIPE) + tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|') + for member in tar.getmembers(): + if not member.isdir(): + name = normpath(member.name) + # enforce proper utf-8 encoding + try: + name.decode('utf-8') + except UnicodeDecodeError: + name = name.decode('iso8859-1').encode('utf-8') + yield name + tar.close() + dpkg.stdout.close() + dpkg.wait() + + def read_control(self): + ''' + Reads the control information from a binary. + + @rtype: text + @return: stanza text of the control section. + ''' + import utils + fullpath = self.poolfile.fullpath + with open(fullpath, 'r') as deb_file: + return utils.deb_extract_control(deb_file) + + def read_control_fields(self): + ''' + Reads the control information from a binary and return + as a dictionary. + + @rtype: dict + @return: fields of the control section as a dictionary. + ''' + stanza = self.read_control() + return apt_pkg.TagSection(stanza) + + @property + def proxy(self): + session = object_session(self) + query = session.query(BinaryMetadata).filter_by(binary=self) + return MetadataProxy(session, query) __all__.append('DBBinary') @@ -274,164 +585,66 @@ def get_suites_binary_in(package, session=None): """ Returns list of Suite objects which given C{package} name is in - @type source: str - @param source: DBBinary package name to search for + @type package: str + @param package: DBBinary package name to search for @rtype: list @return: list of Suite objects for the given package """ - return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all() + return session.query(Suite).filter(Suite.binaries.any(DBBinary.package == package)).all() __all__.append('get_suites_binary_in') @session_wrapper -def get_binary_from_id(id, session=None): - """ - Returns DBBinary object for given C{id} - - @type id: int - @param id: Id of the required binary - - @type session: Session - @param session: Optional SQLA session object (a temporary one will be - generated if not supplied) - - @rtype: DBBinary - @return: DBBinary object for the given binary (None if not present) - """ - - q = session.query(DBBinary).filter_by(binary_id=id) - - try: - return q.one() - except NoResultFound: - return None - -__all__.append('get_binary_from_id') - -@session_wrapper -def get_binaries_from_name(package, version=None, architecture=None, session=None): - """ - Returns list of DBBinary objects for given C{package} name +def get_component_by_package_suite(package, suite_list, arch_list=[], session=None): + ''' + Returns the component name of the newest binary package in suite_list or + None if no package is found. The result can be optionally filtered by a list + of architecture names. @type package: str @param package: DBBinary package name to search for - @type version: str or None - @param version: Version to search for (or None) - - @type package: str, list or None - @param package: Architectures to limit to (or None if no limit) - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of DBBinary objects for the given name (may be empty) - """ - - q = session.query(DBBinary).filter_by(package=package) - - if version is not None: - q = q.filter_by(version=version) - - if architecture is not None: - if not isinstance(architecture, list): - architecture = [architecture] - q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture)) - - ret = q.all() - - return ret - -__all__.append('get_binaries_from_name') - -@session_wrapper -def get_binaries_from_source_id(source_id, session=None): - """ - Returns list of DBBinary objects for given C{source_id} - - @type source_id: int - @param source_id: source_id to search for + @type suite_list: list of str + @param suite_list: list of suite_name items - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of DBBinary objects for the given name (may be empty) - """ - - return session.query(DBBinary).filter_by(source_id=source_id).all() - -__all__.append('get_binaries_from_source_id') - -@session_wrapper -def get_binary_from_name_suite(package, suitename, session=None): - ### For dak examine-package - ### XXX: Doesn't use object API yet - - sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name - FROM binaries b, files fi, location l, component c, bin_associations ba, suite su - WHERE b.package=:package - AND b.file = fi.id - AND fi.location = l.id - AND l.component = c.id - AND ba.bin=b.id - AND ba.suite = su.id - AND su.suite_name=:suitename - ORDER BY b.version DESC""" - - return session.execute(sql, {'package': package, 'suitename': suitename}) - -__all__.append('get_binary_from_name_suite') - -@session_wrapper -def get_binary_components(package, suitename, arch, session=None): - # Check for packages that have moved from one component to another - query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f - WHERE b.package=:package AND s.suite_name=:suitename - AND (a.arch_string = :arch OR a.arch_string = 'all') - AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id - AND f.location = l.id - AND l.component = c.id - AND b.file = f.id""" + @type arch_list: list of str + @param arch_list: optional list of arch_string items that defaults to [] - vals = {'package': package, 'suitename': suitename, 'arch': arch} + @rtype: str or NoneType + @return: name of component or None + ''' - return session.execute(query, vals) - -__all__.append('get_binary_components') - -################################################################################ - -class BinaryACL(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.binary_acl_id + q = session.query(DBBinary).filter_by(package = package). \ + join(DBBinary.suites).filter(Suite.suite_name.in_(suite_list)) + if len(arch_list) > 0: + q = q.join(DBBinary.architecture). \ + filter(Architecture.arch_string.in_(arch_list)) + binary = q.order_by(desc(DBBinary.version)).first() + if binary is None: + return None + else: + return binary.poolfile.component.component_name -__all__.append('BinaryACL') +__all__.append('get_component_by_package_suite') ################################################################################ -class BinaryACLMap(object): +class BuildQueue(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.binary_acl_map_id + return '' % self.queue_name -__all__.append('BinaryACLMap') +__all__.append('BuildQueue') ################################################################################ -class Component(object): - def __init__(self, *args, **kwargs): - pass +class Component(ORMObject): + def __init__(self, component_name = None): + self.component_name = component_name def __eq__(self, val): if isinstance(val, str): @@ -445,8 +658,12 @@ class Component(object): # This signals to use the normal comparison operator return NotImplemented - def __repr__(self): - return '' % self.component_name + def properties(self): + return ['component_name', 'component_id', 'description', \ + 'meets_dfsg', 'overrides_count'] + + def not_null_constraints(self): + return ['component_name'] __all__.append('Component') @@ -474,206 +691,63 @@ def get_component(component, session=None): __all__.append('get_component') -################################################################################ - -class DBConfig(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.name - -__all__.append('DBConfig') - -################################################################################ - -@session_wrapper -def get_or_set_contents_file_id(filename, session=None): - """ - Returns database id for given filename. - - If no matching file is found, a row is inserted. - - @type filename: string - @param filename: The filename - @type session: SQLAlchemy - @param session: Optional SQL session object (a temporary one will be - generated if not supplied). If not passed, a commit will be performed at - the end of the function, otherwise the caller is responsible for commiting. - - @rtype: int - @return: the database id for the given component - """ - - q = session.query(ContentFilename).filter_by(filename=filename) - - try: - ret = q.one().cafilename_id - except NoResultFound: - cf = ContentFilename() - cf.filename = filename - session.add(cf) - session.commit_or_flush() - ret = cf.cafilename_id - - return ret +def get_mapped_component_name(component_name): + cnf = Config() + for m in cnf.value_list("ComponentMappings"): + (src, dst) = m.split() + if component_name == src: + component_name = dst + return component_name -__all__.append('get_or_set_contents_file_id') +__all__.append('get_mapped_component_name') @session_wrapper -def get_contents(suite, overridetype, section=None, session=None): - """ - Returns contents for a suite / overridetype combination, limiting - to a section if not None. +def get_mapped_component(component_name, session=None): + """get component after mappings - @type suite: Suite - @param suite: Suite object + Evaluate component mappings from ComponentMappings in dak.conf for the + given component name. - @type overridetype: OverrideType - @param overridetype: OverrideType object + @todo: ansgar wants to get rid of this. It's currently only used for + the security archive - @type section: Section - @param section: Optional section object to limit results to + @type component_name: str + @param component_name: component name - @type session: SQLAlchemy - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) + @param session: database session - @rtype: ResultsProxy - @return: ResultsProxy object set up to return tuples of (filename, section, - package, arch_id) + @rtype: L{daklib.dbconn.Component} or C{None} + @return: component after applying maps or C{None} """ + component_name = get_mapped_component_name(component_name) + component = session.query(Component).filter_by(component_name=component_name).first() + return component - # find me all of the contents for a given suite - contents_q = """SELECT (p.path||'/'||n.file) AS fn, - s.section, - b.package, - b.architecture - FROM content_associations c join content_file_paths p ON (c.filepath=p.id) - JOIN content_file_names n ON (c.filename=n.id) - JOIN binaries b ON (b.id=c.binary_pkg) - JOIN override o ON (o.package=b.package) - JOIN section s ON (s.id=o.section) - WHERE o.suite = :suiteid AND o.type = :overridetypeid - AND b.type=:overridetypename""" - - vals = {'suiteid': suite.suite_id, - 'overridetypeid': overridetype.overridetype_id, - 'overridetypename': overridetype.overridetype} - - if section is not None: - contents_q += " AND s.id = :sectionid" - vals['sectionid'] = section.section_id - - contents_q += " ORDER BY fn" - - return session.execute(contents_q, vals) - -__all__.append('get_contents') - -################################################################################ - -class ContentFilepath(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.filepath - -__all__.append('ContentFilepath') +__all__.append('get_mapped_component') @session_wrapper -def get_or_set_contents_path_id(filepath, session=None): +def get_component_names(session=None): """ - Returns database id for given path. - - If no matching file is found, a row is inserted. + Returns list of strings of component names. - @type filename: string - @param filename: The filepath - @type session: SQLAlchemy - @param session: Optional SQL session object (a temporary one will be - generated if not supplied). If not passed, a commit will be performed at - the end of the function, otherwise the caller is responsible for commiting. - - @rtype: int - @return: the database id for the given path + @rtype: list + @return: list of strings of component names """ - q = session.query(ContentFilepath).filter_by(filepath=filepath) - - try: - ret = q.one().cafilepath_id - except NoResultFound: - cf = ContentFilepath() - cf.filepath = filepath - session.add(cf) - session.commit_or_flush() - ret = cf.cafilepath_id - - return ret + return [ x.component_name for x in session.query(Component).all() ] -__all__.append('get_or_set_contents_path_id') +__all__.append('get_component_names') ################################################################################ -class ContentAssociation(object): +class DBConfig(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.ca_id - -__all__.append('ContentAssociation') - -def insert_content_paths(binary_id, fullpaths, session=None): - """ - Make sure given path is associated with given binary id - - @type binary_id: int - @param binary_id: the id of the binary - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the caller - is responsible for ensuring a transaction has begun and committing the - results or rolling back based on the result code. If not passed, a commit - will be performed at the end of the function, otherwise the caller is - responsible for commiting. - - @return: True upon success - """ - - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - - try: - # Insert paths - pathcache = {} - for fullpath in fullpaths: - if fullpath.startswith( './' ): - fullpath = fullpath[2:] - - session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} ) - - session.commit() - if privatetrans: - session.close() - return True - - except: - traceback.print_exc() - - # Only rollback if we set up the session ourself - if privatetrans: - session.rollback() - session.close() - - return False + return '' % self.name -__all__.append('insert_content_paths') +__all__.append('DBConfig') ################################################################################ @@ -721,138 +795,85 @@ __all__.append('get_dscfiles') ################################################################################ -class PoolFile(object): +class ExternalOverride(ORMObject): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.filename - -__all__.append('PoolFile') - -@session_wrapper -def check_poolfile(filename, filesize, md5sum, location_id, session=None): - """ - Returns a tuple: - (ValidFileFound [boolean or None], PoolFile object or None) - - @type filename: string - @param filename: the filename of the file to check against the DB - - @type filesize: int - @param filesize: the size of the file to check against the DB - - @type md5sum: string - @param md5sum: the md5sum of the file to check against the DB - - @type location_id: int - @param location_id: the id of the location to look in - - @rtype: tuple - @return: Tuple of length 2. - If more than one file found with that name: - (None, None) - If valid pool file found: (True, PoolFile object) - If valid pool file not found: - (False, None) if no file found - (False, PoolFile object) if file found with size/md5sum mismatch - """ - - q = session.query(PoolFile).filter_by(filename=filename) - q = q.join(Location).filter_by(location_id=location_id) - - ret = None - - if q.count() > 1: - ret = (None, None) - elif q.count() < 1: - ret = (False, None) - else: - obj = q.one() - if obj.md5sum != md5sum or obj.filesize != filesize: - ret = (False, obj) - - if ret is None: - ret = (True, obj) - - return ret - -__all__.append('check_poolfile') - -@session_wrapper -def get_poolfile_by_id(file_id, session=None): - """ - Returns a PoolFile objects or None for the given id - - @type file_id: int - @param file_id: the id of the file to look for - - @rtype: PoolFile or None - @return: either the PoolFile object or None - """ - - q = session.query(PoolFile).filter_by(file_id=file_id) - - try: - return q.one() - except NoResultFound: - return None - -__all__.append('get_poolfile_by_id') - - -@session_wrapper -def get_poolfile_by_name(filename, location_id=None, session=None): - """ - Returns an array of PoolFile objects for the given filename and - (optionally) location_id - - @type filename: string - @param filename: the filename of the file to check against the DB - - @type location_id: int - @param location_id: the id of the location to look in (optional) + return '' % (self.package, self.key, self.value) - @rtype: array - @return: array of PoolFile objects - """ - - q = session.query(PoolFile).filter_by(filename=filename) +__all__.append('ExternalOverride') - if location_id is not None: - q = q.join(Location).filter_by(location_id=location_id) - - return q.all() - -__all__.append('get_poolfile_by_name') - -@session_wrapper -def get_poolfile_like_name(filename, session=None): - """ - Returns an array of PoolFile objects which are like the given name +################################################################################ - @type filename: string - @param filename: the filename of the file to check against the DB +class PoolFile(ORMObject): + def __init__(self, filename = None, filesize = -1, \ + md5sum = None): + self.filename = filename + self.filesize = filesize + self.md5sum = md5sum + + @property + def fullpath(self): + session = DBConn().session().object_session(self) + af = session.query(ArchiveFile).join(Archive) \ + .filter(ArchiveFile.file == self) \ + .order_by(Archive.tainted.desc()).first() + return af.path + + @property + def component(self): + session = DBConn().session().object_session(self) + component_id = session.query(ArchiveFile.component_id).filter(ArchiveFile.file == self) \ + .group_by(ArchiveFile.component_id).one() + return session.query(Component).get(component_id) + + @property + def basename(self): + return os.path.basename(self.filename) + + def is_valid(self, filesize = -1, md5sum = None): + return self.filesize == long(filesize) and self.md5sum == md5sum + + def properties(self): + return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \ + 'sha256sum', 'source', 'binary', 'last_used'] + + def not_null_constraints(self): + return ['filename', 'md5sum'] + + def identical_to(self, filename): + """ + compare size and hash with the given file - @rtype: array - @return: array of PoolFile objects - """ + @rtype: bool + @return: true if the given file has the same size and hash as this object; false otherwise + """ + st = os.stat(filename) + if self.filesize != st.st_size: + return False - # TODO: There must be a way of properly using bind parameters with %FOO% - q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename)) + f = open(filename, "r") + sha256sum = apt_pkg.sha256sum(f) + if sha256sum != self.sha256sum: + return False - return q.all() + return True -__all__.append('get_poolfile_like_name') +__all__.append('PoolFile') ################################################################################ -class Fingerprint(object): - def __init__(self, *args, **kwargs): - pass +class Fingerprint(ORMObject): + def __init__(self, fingerprint = None): + self.fingerprint = fingerprint - def __repr__(self): - return '' % self.fingerprint + def properties(self): + return ['fingerprint', 'fingerprint_id', 'keyring', 'uid', \ + 'binary_reject'] + + def not_null_constraints(self): + return ['fingerprint'] __all__.append('Fingerprint') @@ -932,9 +953,6 @@ def get_ldap_name(entry): ################################################################################ class Keyring(object): - gpg_invocation = "gpg --no-default-keyring --keyring %s" +\ - " --with-colons --fingerprint --fingerprint" - keys = {} fpr_lookup = {} @@ -944,47 +962,56 @@ class Keyring(object): def __repr__(self): return '' % self.keyring_name - def de_escape_gpg_str(self, str): - esclist = re.split(r'(\\x..)', str) + def de_escape_gpg_str(self, txt): + esclist = re.split(r'(\\x..)', txt) for x in range(1,len(esclist),2): esclist[x] = "%c" % (int(esclist[x][2:],16)) return "".join(esclist) - def load_keys(self, keyring): + def parse_address(self, uid): + """parses uid and returns a tuple of real name and email address""" import email.Utils + (name, address) = email.Utils.parseaddr(uid) + name = re.sub(r"\s*[(].*[)]", "", name) + name = self.de_escape_gpg_str(name) + if name == "": + name = uid + return (name, address) + def load_keys(self, keyring): if not self.keyring_id: raise Exception('Must be initialized with database information') - k = os.popen(self.gpg_invocation % keyring, "r") + cmd = ["gpg", "--no-default-keyring", "--keyring", keyring, + "--with-colons", "--fingerprint", "--fingerprint"] + p = daklib.daksubprocess.Popen(cmd, stdout=subprocess.PIPE) + key = None - signingkey = False + need_fingerprint = False - for line in k.xreadlines(): + for line in p.stdout: field = line.split(":") if field[0] == "pub": key = field[4] - (name, addr) = email.Utils.parseaddr(field[9]) - name = re.sub(r"\s*[(].*[)]", "", name) - if name == "" or addr == "" or "@" not in addr: - name = field[9] - addr = "invalid-uid" - name = self.de_escape_gpg_str(name) - self.keys[key] = {"email": addr} - if name != "": + self.keys[key] = {} + (name, addr) = self.parse_address(field[9]) + if "@" in addr: + self.keys[key]["email"] = addr self.keys[key]["name"] = name - self.keys[key]["aliases"] = [name] - self.keys[key]["fingerprints"] = [] - signingkey = True - elif key and field[0] == "sub" and len(field) >= 12: - signingkey = ("s" in field[11]) + need_fingerprint = True elif key and field[0] == "uid": - (name, addr) = email.Utils.parseaddr(field[9]) - if name and name not in self.keys[key]["aliases"]: - self.keys[key]["aliases"].append(name) - elif signingkey and field[0] == "fpr": - self.keys[key]["fingerprints"].append(field[9]) + (name, addr) = self.parse_address(field[9]) + if "email" not in self.keys[key] and "@" in addr: + self.keys[key]["email"] = addr + self.keys[key]["name"] = name + elif need_fingerprint and field[0] == "fpr": + self.keys[key]["fingerprints"] = [field[9]] self.fpr_lookup[field[9]] = key + need_fingerprint = False + + r = p.wait() + if r != 0: + raise subprocess.CalledProcessError(r, cmd) def import_users_from_ldap(self, session): import ldap @@ -992,11 +1019,19 @@ class Keyring(object): LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"] LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"] + ca_cert_file = cnf.get('Import-LDAP-Fingerprints::CACertFile') l = ldap.open(LDAPServer) + + if ca_cert_file: + l.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_HARD) + l.set_option(ldap.OPT_X_TLS_CACERTFILE, ca_cert_file) + l.set_option(ldap.OPT_X_TLS_NEWCTX, True) + l.start_tls_s() + l.simple_bind_s("","") Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL, - "(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]), + "(&(keyfingerprint=*)(supplementaryGid=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]), ["uid", "keyfingerprint", "cn", "mn", "sn"]) ldap_fin_uid_id = {} @@ -1029,7 +1064,7 @@ class Keyring(object): byname = {} any_invalid = False for x in self.keys.keys(): - if self.keys[x]["email"] == "invalid-uid": + if "email" not in self.keys[x]: any_invalid = True self.keys[x]["uid"] = format % "invalid-uid" else: @@ -1071,105 +1106,63 @@ def get_keyring(keyring, session=None): __all__.append('get_keyring') -################################################################################ - -class KeyringACLMap(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.keyring_acl_map_id +@session_wrapper +def get_active_keyring_paths(session=None): + """ + @rtype: list + @return: list of active keyring paths + """ + return [ x.keyring_name for x in session.query(Keyring).filter(Keyring.active == True).order_by(desc(Keyring.priority)).all() ] -__all__.append('KeyringACLMap') +__all__.append('get_active_keyring_paths') ################################################################################ -class KnownChange(object): +class DBChange(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.changesname + return '' % self.changesname -__all__.append('KnownChange') +__all__.append('DBChange') @session_wrapper -def get_knownchange(filename, session=None): +def get_dbchange(filename, session=None): """ - returns knownchange object for given C{filename}. + returns DBChange object for given C{filename}. - @type archive: string - @param archive: the name of the arhive + @type filename: string + @param filename: the name of the file @type session: Session @param session: Optional SQLA session object (a temporary one will be generated if not supplied) - @rtype: Archive - @return: Archive object for the given name (None if not present) + @rtype: DBChange + @return: DBChange object for the given filename (C{None} if not present) """ - q = session.query(KnownChange).filter_by(changesname=filename) + q = session.query(DBChange).filter_by(changesname=filename) try: return q.one() except NoResultFound: return None -__all__.append('get_knownchange') +__all__.append('get_dbchange') ################################################################################ -class Location(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.path, self.location_id) - -__all__.append('Location') - -@session_wrapper -def get_location(location, component=None, archive=None, session=None): - """ - Returns Location object for the given combination of location, component - and archive - - @type location: string - @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/} - - @type component: string - @param component: the component name (if None, no restriction applied) - - @type archive: string - @param archive_id: the archive name (if None, no restriction applied) - @rtype: Location / None - @return: Either a Location object or None if one can't be found - """ - - q = session.query(Location).filter_by(path=location) - - if archive is not None: - q = q.join(Archive).filter_by(archive_name=archive) - - if component is not None: - q = q.join(Component).filter_by(component_name=component) +class Maintainer(ORMObject): + def __init__(self, name = None): + self.name = name - try: - return q.one() - except NoResultFound: - return None - -__all__.append('get_location') - -################################################################################ - -class Maintainer(object): - def __init__(self, *args, **kwargs): - pass + def properties(self): + return ['name', 'maintainer_id'] - def __repr__(self): - return '''''' % (self.name, self.maintainer_id) + def not_null_constraints(self): + return ['name'] def get_split_maintainer(self): if not hasattr(self, 'name') or self.name is None: @@ -1242,7 +1235,7 @@ class NewComment(object): __all__.append('NewComment') @session_wrapper -def has_new_comment(package, version, session=None): +def has_new_comment(policy_queue, package, version, session=None): """ Returns true if the given combination of C{package}, C{version} has a comment. @@ -1260,7 +1253,7 @@ def has_new_comment(package, version, session=None): @return: true/false """ - q = session.query(NewComment) + q = session.query(NewComment).filter_by(policy_queue=policy_queue) q = q.filter_by(package=package) q = q.filter_by(version=version) @@ -1269,7 +1262,7 @@ def has_new_comment(package, version, session=None): __all__.append('has_new_comment') @session_wrapper -def get_new_comments(package=None, version=None, comment_id=None, session=None): +def get_new_comments(policy_queue, package=None, version=None, comment_id=None, session=None): """ Returns (possibly empty) list of NewComment objects for the given parameters @@ -1291,7 +1284,7 @@ def get_new_comments(package=None, version=None, comment_id=None, session=None): @return: A (possibly empty) list of NewComment objects will be returned """ - q = session.query(NewComment) + q = session.query(NewComment).filter_by(policy_queue=policy_queue) if package is not None: q = q.filter_by(package=package) if version is not None: q = q.filter_by(version=version) if comment_id is not None: q = q.filter_by(comment_id=comment_id) @@ -1302,12 +1295,22 @@ __all__.append('get_new_comments') ################################################################################ -class Override(object): - def __init__(self, *args, **kwargs): - pass +class Override(ORMObject): + def __init__(self, package = None, suite = None, component = None, overridetype = None, \ + section = None, priority = None): + self.package = package + self.suite = suite + self.component = component + self.overridetype = overridetype + self.section = section + self.priority = priority - def __repr__(self): - return '' % (self.package, self.suite_id) + def properties(self): + return ['package', 'suite', 'component', 'overridetype', 'section', \ + 'priority'] + + def not_null_constraints(self): + return ['package', 'suite', 'component', 'overridetype', 'section'] __all__.append('Override') @@ -1361,12 +1364,15 @@ __all__.append('get_override') ################################################################################ -class OverrideType(object): - def __init__(self, *args, **kwargs): - pass +class OverrideType(ORMObject): + def __init__(self, overridetype = None): + self.overridetype = overridetype - def __repr__(self): - return '' % self.overridetype + def properties(self): + return ['overridetype', 'overridetype_id', 'overrides_count'] + + def not_null_constraints(self): + return ['overridetype'] __all__.append('OverrideType') @@ -1397,97 +1403,77 @@ __all__.append('get_override_type') ################################################################################ -class PendingContentAssociation(object): +class PolicyQueue(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.pca_id + return '' % self.queue_name -__all__.append('PendingContentAssociation') +__all__.append('PolicyQueue') -def insert_pending_content_paths(package, fullpaths, session=None): +@session_wrapper +def get_policy_queue(queuename, session=None): """ - Make sure given paths are temporarily associated with given - package + Returns PolicyQueue object for given C{queue name} - @type package: dict - @param package: the package to associate with should have been read in from the binary control file - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the caller - is responsible for ensuring a transaction has begun and committing the - results or rolling back based on the result code. If not passed, a commit - will be performed at the end of the function + @type queuename: string + @param queuename: The name of the queue - @return: True upon success, False if there is a problem - """ + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) - privatetrans = False + @rtype: PolicyQueue + @return: PolicyQueue object for the given queue + """ - if session is None: - session = DBConn().session() - privatetrans = True + q = session.query(PolicyQueue).filter_by(queue_name=queuename) try: - arch = get_architecture(package['Architecture'], session) - arch_id = arch.arch_id - - # Remove any already existing recorded files for this package - q = session.query(PendingContentAssociation) - q = q.filter_by(package=package['Package']) - q = q.filter_by(version=package['Version']) - q = q.filter_by(architecture=arch_id) - q.delete() - - # Insert paths - pathcache = {} - for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) - - if path.startswith( "./" ): - path = path[2:] - - filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) - - pathcache[fullpath] = (filepath_id, filename_id) - - for fullpath, dat in pathcache.items(): - pca = PendingContentAssociation() - pca.package = package['Package'] - pca.version = package['Version'] - pca.filepath_id = dat[0] - pca.filename_id = dat[1] - pca.architecture = arch_id - session.add(pca) - - # Only commit if we set up the session ourself - if privatetrans: - session.commit() - session.close() - else: - session.flush() + return q.one() + except NoResultFound: + return None - return True - except Exception, e: - traceback.print_exc() +__all__.append('get_policy_queue') - # Only rollback if we set up the session ourself - if privatetrans: - session.rollback() - session.close() +################################################################################ - return False +class PolicyQueueUpload(object): + def __cmp__(self, other): + ret = cmp(self.changes.source, other.changes.source) + if ret == 0: + ret = apt_pkg.version_compare(self.changes.version, other.changes.version) + if ret == 0: + if self.source is not None and other.source is None: + ret = -1 + elif self.source is None and other.source is not None: + ret = 1 + if ret == 0: + ret = cmp(self.changes.changesname, other.changes.changesname) + return ret + +__all__.append('PolicyQueueUpload') + +################################################################################ + +class PolicyQueueByhandFile(object): + pass -__all__.append('insert_pending_content_paths') +__all__.append('PolicyQueueByhandFile') ################################################################################ -class Priority(object): - def __init__(self, *args, **kwargs): - pass +class Priority(ORMObject): + def __init__(self, priority = None, level = None): + self.priority = priority + self.level = level + + def properties(self): + return ['priority', 'priority_id', 'level', 'overrides_count'] + + def not_null_constraints(self): + return ['priority', 'level'] def __eq__(self, val): if isinstance(val, str): @@ -1501,9 +1487,6 @@ class Priority(object): # This signals to use the normal comparison operator return NotImplemented - def __repr__(self): - return '' % (self.priority, self.priority_id) - __all__.append('Priority') @session_wrapper @@ -1555,212 +1538,15 @@ __all__.append('get_priorities') ################################################################################ -class Queue(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.queue_name - - def autobuild_upload(self, changes, srcpath, session=None): - """ - Update queue_build database table used for incoming autobuild support. - - @type changes: Changes - @param changes: changes object for the upload to process - - @type srcpath: string - @param srcpath: path for the queue file entries/link destinations - - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the - caller is responsible for ensuring a transaction has begun and - committing the results or rolling back based on the result code. If - not passed, a commit will be performed at the end of the function, - otherwise the caller is responsible for commiting. - - @rtype: NoneType or string - @return: None if the operation failed, a string describing the error if not - """ - - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - - # TODO: Remove by moving queue config into the database - conf = Config() - - for suitename in changes.changes["distribution"].keys(): - # TODO: Move into database as: - # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build) - # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink) - # This also gets rid of the SecurityQueueBuild hack below - if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"): - continue - - # Find suite object - s = get_suite(suitename, session) - if s is None: - return "INTERNAL ERROR: Could not find suite %s" % suitename - - # TODO: Get from database as above - dest_dir = conf["Dir::QueueBuild"] - - # TODO: Move into database as above - if conf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suitename) - - for file_entry in changes.files.keys(): - src = os.path.join(srcpath, file_entry) - dest = os.path.join(dest_dir, file_entry) - - # TODO: Move into database as above - if conf.FindB("Dinstall::SecurityQueueBuild"): - # Copy it since the original won't be readable by www-data - import utils - utils.copy(src, dest) - else: - # Create a symlink to it - os.symlink(src, dest) - - qb = QueueBuild() - qb.suite_id = s.suite_id - qb.queue_id = self.queue_id - qb.filename = dest - qb.in_queue = True - - session.add(qb) - - # If the .orig tarballs are in the pool, create a symlink to - # them (if one doesn't already exist) - for dsc_file in changes.dsc_files.keys(): - # Skip all files except orig tarballs - from daklib.regexes import re_is_orig_source - if not re_is_orig_source.match(dsc_file): - continue - # Skip orig files not identified in the pool - if not (changes.orig_files.has_key(dsc_file) and - changes.orig_files[dsc_file].has_key("id")): - continue - orig_file_id = changes.orig_files[dsc_file]["id"] - dest = os.path.join(dest_dir, dsc_file) - - # If it doesn't exist, create a symlink - if not os.path.exists(dest): - q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", - {'id': orig_file_id}) - res = q.fetchone() - if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id) - - src = os.path.join(res[0], res[1]) - os.symlink(src, dest) - - # Add it to the list of packages for later processing by apt-ftparchive - qb = QueueBuild() - qb.suite_id = s.suite_id - qb.queue_id = self.queue_id - qb.filename = dest - qb.in_queue = True - session.add(qb) - - # If it does, update things to ensure it's not removed prematurely - else: - qb = get_queue_build(dest, s.suite_id, session) - if qb is None: - qb.in_queue = True - qb.last_used = None - session.add(qb) - - if privatetrans: - session.commit() - session.close() - - return None - -__all__.append('Queue') - -@session_wrapper -def get_or_set_queue(queuename, session=None): - """ - Returns Queue object for given C{queue name}, creating it if it does not - exist. - - @type queuename: string - @param queuename: The name of the queue - - @type session: Session - @param session: Optional SQLA session object (a temporary one will be - generated if not supplied) - - @rtype: Queue - @return: Queue object for the given queue - """ - - q = session.query(Queue).filter_by(queue_name=queuename) - - try: - ret = q.one() - except NoResultFound: - queue = Queue() - queue.queue_name = queuename - session.add(queue) - session.commit_or_flush() - ret = queue - - return ret - -__all__.append('get_or_set_queue') - -################################################################################ - -class QueueBuild(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.filename, self.queue_id) - -__all__.append('QueueBuild') - -@session_wrapper -def get_queue_build(filename, suite, session=None): - """ - Returns QueueBuild object for given C{filename} and C{suite}. - - @type filename: string - @param filename: The name of the file - - @type suiteid: int or str - @param suiteid: Suite name or ID - - @type session: Session - @param session: Optional SQLA session object (a temporary one will be - generated if not supplied) - - @rtype: Queue - @return: Queue object for the given queue - """ - - if isinstance(suite, int): - q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite) - else: - q = session.query(QueueBuild).filter_by(filename=filename) - q = q.join(Suite).filter_by(suite_name=suite) - - try: - return q.one() - except NoResultFound: - return None +class Section(ORMObject): + def __init__(self, section = None): + self.section = section -__all__.append('get_queue_build') + def properties(self): + return ['section', 'section_id', 'overrides_count'] -################################################################################ - -class Section(object): - def __init__(self, *args, **kwargs): - pass + def not_null_constraints(self): + return ['section'] def __eq__(self, val): if isinstance(val, str): @@ -1774,9 +1560,6 @@ class Section(object): # This signals to use the normal comparison operator return NotImplemented - def __repr__(self): - return '
' % self.section - __all__.append('Section') @session_wrapper @@ -1817,93 +1600,169 @@ def get_sections(session=None): @return: dictionary of section names -> id mappings """ - ret = {} - q = session.query(Section) - for x in q.all(): - ret[x.section] = x.section_id + ret = {} + q = session.query(Section) + for x in q.all(): + ret[x.section] = x.section_id + + return ret + +__all__.append('get_sections') + +################################################################################ + +class SignatureHistory(ORMObject): + @classmethod + def from_signed_file(cls, signed_file): + """signature history entry from signed file + + @type signed_file: L{daklib.gpg.SignedFile} + @param signed_file: signed file + + @rtype: L{SignatureHistory} + """ + self = cls() + self.fingerprint = signed_file.primary_fingerprint + self.signature_timestamp = signed_file.signature_timestamp + self.contents_sha1 = signed_file.contents_sha1() + return self + + def query(self, session): + return session.query(SignatureHistory).filter_by(fingerprint=self.fingerprint, signature_timestamp=self.signature_timestamp, contents_sha1=self.contents_sha1).first() + +__all__.append('SignatureHistory') + +################################################################################ + +class SrcContents(ORMObject): + def __init__(self, file = None, source = None): + self.file = file + self.source = source - return ret + def properties(self): + return ['file', 'source'] -__all__.append('get_sections') +__all__.append('SrcContents') ################################################################################ -class DBSource(object): - def __init__(self, *args, **kwargs): - pass +from debian.debfile import Deb822 - def __repr__(self): - return '' % (self.source, self.version) +# Temporary Deb822 subclass to fix bugs with : handling; see #597249 +class Dak822(Deb822): + def _internal_parser(self, sequence, fields=None): + # The key is non-whitespace, non-colon characters before any colon. + key_part = r"^(?P[^: \t\n\r\f\v]+)\s*:\s*" + single = re.compile(key_part + r"(?P\S.*?)\s*$") + multi = re.compile(key_part + r"$") + multidata = re.compile(r"^\s(?P.+?)\s*$") -__all__.append('DBSource') + wanted_field = lambda f: fields is None or f in fields -@session_wrapper -def source_exists(source, source_version, suites = ["any"], session=None): - """ - Ensure that source exists somewhere in the archive for the binary - upload being processed. - 1. exact match => 1.0-3 - 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 + if isinstance(sequence, basestring): + sequence = sequence.splitlines() - @type package: string - @param package: package source name + curkey = None + content = "" + for line in self.gpg_stripped_paragraph(sequence): + m = single.match(line) + if m: + if curkey: + self[curkey] = content - @type source_version: string - @param source_version: expected source version + if not wanted_field(m.group('key')): + curkey = None + continue - @type suites: list - @param suites: list of suites to check in, default I{any} + curkey = m.group('key') + content = m.group('data') + continue - @type session: Session - @param session: Optional SQLA session object (a temporary one will be - generated if not supplied) + m = multi.match(line) + if m: + if curkey: + self[curkey] = content - @rtype: int - @return: returns 1 if a source with expected version is found, otherwise 0 + if not wanted_field(m.group('key')): + curkey = None + continue - """ + curkey = m.group('key') + content = "" + continue - cnf = Config() - ret = 1 - - for suite in suites: - q = session.query(DBSource).filter_by(source=source) - if suite != "any": - # source must exist in suite X, or in some other suite that's - # mapped to X, recursively... silent-maps are counted too, - # unreleased-maps aren't. - maps = cnf.ValueList("SuiteMappings")[:] - maps.reverse() - maps = [ m.split() for m in maps ] - maps = [ (x[1], x[2]) for x in maps - if x[0] == "map" or x[0] == "silent-map" ] - s = [suite] - for x in maps: - if x[1] in s and x[0] not in s: - s.append(x[0]) - - q = q.join(SrcAssociation).join(Suite) - q = q.filter(Suite.suite_name.in_(s)) - - # Reduce the query results to a list of version numbers - ql = [ j.version for j in q.all() ] - - # Try (1) - if source_version in ql: - continue - - # Try (2) - from daklib.regexes import re_bin_only_nmu - orig_source_version = re_bin_only_nmu.sub('', source_version) - if orig_source_version in ql: - continue - - # No source found so return not ok - ret = 0 + m = multidata.match(line) + if m: + content += '\n' + line # XXX not m.group('data')? + continue - return ret + if curkey: + self[curkey] = content + + +class DBSource(ORMObject): + def __init__(self, source = None, version = None, maintainer = None, \ + changedby = None, poolfile = None, install_date = None, fingerprint = None): + self.source = source + self.version = version + self.maintainer = maintainer + self.changedby = changedby + self.poolfile = poolfile + self.install_date = install_date + self.fingerprint = fingerprint + + @property + def pkid(self): + return self.source_id + + def properties(self): + return ['source', 'source_id', 'maintainer', 'changedby', \ + 'fingerprint', 'poolfile', 'version', 'suites_count', \ + 'install_date', 'binaries_count', 'uploaders_count'] + + def not_null_constraints(self): + return ['source', 'version', 'maintainer', \ + 'changedby', 'poolfile'] + + def read_control_fields(self): + ''' + Reads the control information from a dsc + + @rtype: tuple + @return: fields is the dsc information in a dictionary form + ''' + fullpath = self.poolfile.fullpath + fields = Dak822(open(self.poolfile.fullpath, 'r')) + return fields + + metadata = association_proxy('key', 'value') + + def scan_contents(self): + ''' + Returns a set of names for non directories. The path names are + normalized after converting them from either utf-8 or iso8859-1 + encoding. + ''' + fullpath = self.poolfile.fullpath + from daklib.contents import UnpackedSource + unpacked = UnpackedSource(fullpath) + fileset = set() + for name in unpacked.get_all_filenames(): + # enforce proper utf-8 encoding + try: + name.decode('utf-8') + except UnicodeDecodeError: + name = name.decode('iso8859-1').encode('utf-8') + fileset.add(name) + return fileset + + @property + def proxy(self): + session = object_session(self) + query = session.query(SourceMetadata).filter_by(source=self) + return MetadataProxy(session, query) -__all__.append('source_exists') +__all__.append('DBSource') @session_wrapper def get_suites_source_in(source, session=None): @@ -1917,96 +1776,66 @@ def get_suites_source_in(source, session=None): @return: list of Suite objects for the given source """ - return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all() + return session.query(Suite).filter(Suite.sources.any(source=source)).all() __all__.append('get_suites_source_in') +# FIXME: This function fails badly if it finds more than 1 source package and +# its implementation is trivial enough to be inlined. @session_wrapper -def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None): - """ - Returns list of DBSource objects for given C{source} name and other parameters - - @type source: str - @param source: DBSource package name to search for - - @type source: str or None - @param source: DBSource version name to search for or None if not applicable - - @type dm_upload_allowed: bool - @param dm_upload_allowed: If None, no effect. If True or False, only - return packages with that dm_upload_allowed setting - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: list of DBSource objects for the given name (may be empty) - """ - - q = session.query(DBSource).filter_by(source=source) - - if version is not None: - q = q.filter_by(version=version) - - if dm_upload_allowed is not None: - q = q.filter_by(dm_upload_allowed=dm_upload_allowed) - - return q.all() - -__all__.append('get_sources_from_name') - -@session_wrapper -def get_source_in_suite(source, suite, session=None): +def get_source_in_suite(source, suite_name, session=None): """ - Returns list of DBSource objects for a combination of C{source} and C{suite}. + Returns a DBSource object for a combination of C{source} and C{suite_name}. - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} - - B{suite} - a suite name, eg. I{unstable} + - B{suite_name} - a suite name, eg. I{unstable} @type source: string @param source: source package name - @type suite: string + @type suite_name: string @param suite: the suite name @rtype: string @return: the version for I{source} in I{suite} """ - - q = session.query(SrcAssociation) - q = q.join('source').filter_by(source=source) - q = q.join('suite').filter_by(suite_name=suite) - + suite = get_suite(suite_name, session) + if suite is None: + return None try: - return q.one().source + return suite.get_sources(source).one() except NoResultFound: return None __all__.append('get_source_in_suite') -################################################################################ - -class SourceACL(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.source_acl_id - -__all__.append('SourceACL') - -################################################################################ - -class SrcAssociation(object): - def __init__(self, *args, **kwargs): - pass +@session_wrapper +def import_metadata_into_db(obj, session=None): + """ + This routine works on either DBBinary or DBSource objects and imports + their metadata into the database + """ + fields = obj.read_control_fields() + for k in fields.keys(): + try: + # Try raw ASCII + val = str(fields[k]) + except UnicodeEncodeError: + # Fall back to UTF-8 + try: + val = fields[k].encode('utf-8') + except UnicodeEncodeError: + # Finally try iso8859-1 + val = fields[k].encode('iso8859-1') + # Otherwise we allow the exception to percolate up and we cause + # a reject as someone is playing silly buggers + + obj.metadata[get_or_set_metadatakey(k, session)] = val - def __repr__(self): - return '' % (self.sa_id, self.source, self.suite) + session.commit_or_flush() -__all__.append('SrcAssociation') +__all__.append('import_metadata_into_db') ################################################################################ @@ -2021,17 +1850,6 @@ __all__.append('SrcFormat') ################################################################################ -class SrcUploader(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.uploader_id - -__all__.append('SrcUploader') - -################################################################################ - SUITE_FIELDS = [ ('SuiteName', 'suite_name'), ('SuiteID', 'suite_id'), ('Version', 'version'), @@ -2046,18 +1864,21 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'), ('Priority', 'priority'), ('NotAutomatic', 'notautomatic'), ('CopyChanges', 'copychanges'), - ('CopyDotDak', 'copydotdak'), - ('CommentsDir', 'commentsdir'), - ('OverrideSuite', 'overridesuite'), - ('ChangelogBase', 'changelogbase')] + ('OverrideSuite', 'overridesuite')] +# Why the heck don't we have any UNIQUE constraints in table suite? +# TODO: Add UNIQUE constraints for appropriate columns. +class Suite(ORMObject): + def __init__(self, suite_name = None, version = None): + self.suite_name = suite_name + self.version = version -class Suite(object): - def __init__(self, *args, **kwargs): - pass + def properties(self): + return ['suite_name', 'version', 'sources_count', 'binaries_count', \ + 'overrides_count'] - def __repr__(self): - return '' % self.suite_name + def not_null_constraints(self): + return ['suite_name'] def __eq__(self, val): if isinstance(val, str): @@ -2080,38 +1901,67 @@ class Suite(object): return "\n".join(ret) -__all__.append('Suite') + def get_architectures(self, skipsrc=False, skipall=False): + """ + Returns list of Architecture objects -@session_wrapper -def get_suite_architecture(suite, architecture, session=None): - """ - Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it - doesn't exist + @type skipsrc: boolean + @param skipsrc: Whether to skip returning the 'source' architecture entry + (Default False) - @type suite: str - @param suite: Suite name to search for + @type skipall: boolean + @param skipall: Whether to skip returning the 'all' architecture entry + (Default False) - @type architecture: str - @param architecture: Architecture name to search for + @rtype: list + @return: list of Architecture objects for the given name (may be empty) + """ - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) + q = object_session(self).query(Architecture).with_parent(self) + if skipsrc: + q = q.filter(Architecture.arch_string != 'source') + if skipall: + q = q.filter(Architecture.arch_string != 'all') + return q.order_by(Architecture.arch_string).all() - @rtype: SuiteArchitecture - @return: the SuiteArchitecture object or None - """ + def get_sources(self, source): + """ + Returns a query object representing DBSource that is part of C{suite}. - q = session.query(SuiteArchitecture) - q = q.join(Architecture).filter_by(arch_string=architecture) - q = q.join(Suite).filter_by(suite_name=suite) + - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} - try: - return q.one() - except NoResultFound: - return None + @type source: string + @param source: source package name + + @rtype: sqlalchemy.orm.query.Query + @return: a query of DBSource + + """ + + session = object_session(self) + return session.query(DBSource).filter_by(source = source). \ + with_parent(self) + + def get_overridesuite(self): + if self.overridesuite is None: + return self + else: + return object_session(self).query(Suite).filter_by(suite_name=self.overridesuite).one() + + def update_last_changed(self): + self.last_changed = sqlalchemy.func.now() -__all__.append('get_suite_architecture') + @property + def path(self): + return os.path.join(self.archive.path, 'dists', self.suite_name) + + @property + def release_suite_output(self): + if self.release_suite is not None: + return self.release_suite + return self.suite_name + +__all__.append('Suite') @session_wrapper def get_suite(suite, session=None): @@ -2129,8 +1979,22 @@ def get_suite(suite, session=None): @return: Suite object for the requested suite name (None if not present) """ + # Start by looking for the dak internal name q = session.query(Suite).filter_by(suite_name=suite) + try: + return q.one() + except NoResultFound: + pass + + # Now try codename + q = session.query(Suite).filter_by(codename=suite) + try: + return q.one() + except NoResultFound: + pass + # Finally give release_suite a try + q = session.query(Suite).filter_by(release_suite=suite) try: return q.one() except NoResultFound: @@ -2140,22 +2004,14 @@ __all__.append('get_suite') ################################################################################ -class SuiteArchitecture(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.suite_id, self.arch_id) - -__all__.append('SuiteArchitecture') - @session_wrapper def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): """ - Returns list of Architecture objects for given C{suite} name + Returns list of Architecture objects for given C{suite} name. The list is + empty if suite does not exist. - @type source: str - @param source: Suite name to search for + @type suite: str + @param suite: Suite name to search for @type skipsrc: boolean @param skipsrc: Whether to skip returning the 'source' architecture entry @@ -2173,63 +2029,19 @@ def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): @return: list of Architecture objects for the given name (may be empty) """ - q = session.query(Architecture) - q = q.join(SuiteArchitecture) - q = q.join(Suite).filter_by(suite_name=suite) - - if skipsrc: - q = q.filter(Architecture.arch_string != 'source') - - if skipall: - q = q.filter(Architecture.arch_string != 'all') - - q = q.order_by('arch_string') - - return q.all() + try: + return get_suite(suite, session).get_architectures(skipsrc, skipall) + except AttributeError: + return [] __all__.append('get_suite_architectures') ################################################################################ -class SuiteSrcFormat(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.suite_id, self.src_format_id) - -__all__.append('SuiteSrcFormat') - -@session_wrapper -def get_suite_src_formats(suite, session=None): - """ - Returns list of allowed SrcFormat for C{suite}. - - @type suite: str - @param suite: Suite name to search for - - @type session: Session - @param session: Optional SQL session object (a temporary one will be - generated if not supplied) - - @rtype: list - @return: the list of allowed source formats for I{suite} - """ - - q = session.query(SrcFormat) - q = q.join(SuiteSrcFormat) - q = q.join(Suite).filter_by(suite_name=suite) - q = q.order_by('format_name') - - return q.all() - -__all__.append('get_suite_src_formats') - -################################################################################ - -class Uid(object): - def __init__(self, *args, **kwargs): - pass +class Uid(ORMObject): + def __init__(self, uid = None, name = None): + self.uid = uid + self.name = name def __eq__(self, val): if isinstance(val, str): @@ -2243,32 +2055,13 @@ class Uid(object): # This signals to use the normal comparison operator return NotImplemented - def __repr__(self): - return '' % (self.uid, self.name) - -__all__.append('Uid') - -@session_wrapper -def add_database_user(uidname, session=None): - """ - Adds a database user - - @type uidname: string - @param uidname: The uid of the user to add - - @type session: SQLAlchemy - @param session: Optional SQL session object (a temporary one will be - generated if not supplied). If not passed, a commit will be performed at - the end of the function, otherwise the caller is responsible for commiting. - - @rtype: Uid - @return: the uid object for the given uidname - """ + def properties(self): + return ['uid', 'name', 'fingerprint'] - session.execute("CREATE USER :uid", {'uid': uidname}) - session.commit_or_flush() + def not_null_constraints(self): + return ['uid'] -__all__.append('add_database_user') +__all__.append('Uid') @session_wrapper def get_or_set_uid(uidname, session=None): @@ -2318,84 +2111,280 @@ __all__.append('get_uid_from_fingerprint') ################################################################################ -class UploadBlock(object): +class MetadataKey(ORMObject): + def __init__(self, key = None): + self.key = key + + def properties(self): + return ['key'] + + def not_null_constraints(self): + return ['key'] + +__all__.append('MetadataKey') + +@session_wrapper +def get_or_set_metadatakey(keyname, session=None): + """ + Returns MetadataKey object for given uidname. + + If no matching keyname is found, a row is inserted. + + @type uidname: string + @param uidname: The keyname to add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. + + @rtype: MetadataKey + @return: the metadatakey object for the given keyname + """ + + q = session.query(MetadataKey).filter_by(key=keyname) + + try: + ret = q.one() + except NoResultFound: + ret = MetadataKey(keyname) + session.add(ret) + session.commit_or_flush() + + return ret + +__all__.append('get_or_set_metadatakey') + +################################################################################ + +class BinaryMetadata(ORMObject): + def __init__(self, key = None, value = None, binary = None): + self.key = key + self.value = value + if binary is not None: + self.binary = binary + + def properties(self): + return ['binary', 'key', 'value'] + + def not_null_constraints(self): + return ['value'] + +__all__.append('BinaryMetadata') + +################################################################################ + +class SourceMetadata(ORMObject): + def __init__(self, key = None, value = None, source = None): + self.key = key + self.value = value + if source is not None: + self.source = source + + def properties(self): + return ['source', 'key', 'value'] + + def not_null_constraints(self): + return ['value'] + +__all__.append('SourceMetadata') + +################################################################################ + +class MetadataProxy(object): + def __init__(self, session, query): + self.session = session + self.query = query + + def _get(self, key): + metadata_key = self.session.query(MetadataKey).filter_by(key=key).first() + if metadata_key is None: + return None + metadata = self.query.filter_by(key=metadata_key).first() + return metadata + + def __contains__(self, key): + if self._get(key) is not None: + return True + return False + + def __getitem__(self, key): + metadata = self._get(key) + if metadata is None: + raise KeyError + return metadata.value + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + +################################################################################ + +class VersionCheck(ORMObject): def __init__(self, *args, **kwargs): - pass + pass - def __repr__(self): - return '' % (self.source, self.upload_block_id) + def properties(self): + #return ['suite_id', 'check', 'reference_id'] + return ['check'] + + def not_null_constraints(self): + return ['suite', 'check', 'reference'] + +__all__.append('VersionCheck') -__all__.append('UploadBlock') +@session_wrapper +def get_version_checks(suite_name, check = None, session = None): + suite = get_suite(suite_name, session) + if not suite: + # Make sure that what we return is iterable so that list comprehensions + # involving this don't cause a traceback + return [] + q = session.query(VersionCheck).filter_by(suite=suite) + if check: + q = q.filter_by(check=check) + return q.all() + +__all__.append('get_version_checks') ################################################################################ -class DBConn(Singleton): +class DBConn(object): """ database module init. """ + __shared_state = {} + def __init__(self, *args, **kwargs): - super(DBConn, self).__init__(*args, **kwargs) + self.__dict__ = self.__shared_state - def _startup(self, *args, **kwargs): - self.debug = False - if kwargs.has_key('debug'): - self.debug = True - self.__createconn() + if not getattr(self, 'initialised', False): + self.initialised = True + self.debug = kwargs.has_key('debug') + self.__createconn() def __setuptables(self): - self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) - self.tbl_archive = Table('archive', self.db_meta, autoload=True) - self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) - self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) - self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) - self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True) - self.tbl_component = Table('component', self.db_meta, autoload=True) - self.tbl_config = Table('config', self.db_meta, autoload=True) - self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) - self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True) - self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True) - self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True) - self.tbl_files = Table('files', self.db_meta, autoload=True) - self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) - self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) - self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True) - self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True) - self.tbl_location = Table('location', self.db_meta, autoload=True) - self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) - self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) - self.tbl_override = Table('override', self.db_meta, autoload=True) - self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) - self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True) - self.tbl_priority = Table('priority', self.db_meta, autoload=True) - self.tbl_queue = Table('queue', self.db_meta, autoload=True) - self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True) - self.tbl_section = Table('section', self.db_meta, autoload=True) - self.tbl_source = Table('source', self.db_meta, autoload=True) - self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) - self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True) - self.tbl_src_format = Table('src_format', self.db_meta, autoload=True) - self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True) - self.tbl_suite = Table('suite', self.db_meta, autoload=True) - self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) - self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) - self.tbl_uid = Table('uid', self.db_meta, autoload=True) - self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) + tables = ( + 'acl', + 'acl_architecture_map', + 'acl_fingerprint_map', + 'acl_per_source', + 'architecture', + 'archive', + 'bin_associations', + 'bin_contents', + 'binaries', + 'binaries_metadata', + 'build_queue', + 'changelogs_text', + 'changes', + 'component', + 'component_suite', + 'config', + 'dsc_files', + 'external_overrides', + 'extra_src_references', + 'files', + 'files_archive_map', + 'fingerprint', + 'keyrings', + 'maintainer', + 'metadata_keys', + 'new_comments', + # TODO: the maintainer column in table override should be removed. + 'override', + 'override_type', + 'policy_queue', + 'policy_queue_upload', + 'policy_queue_upload_binaries_map', + 'policy_queue_byhand_file', + 'priority', + 'section', + 'signature_history', + 'source', + 'source_metadata', + 'src_associations', + 'src_contents', + 'src_format', + 'src_uploaders', + 'suite', + 'suite_acl_map', + 'suite_architectures', + 'suite_build_queue_copy', + 'suite_src_formats', + 'uid', + 'version_check', + ) + + views = ( + 'almost_obsolete_all_associations', + 'almost_obsolete_src_associations', + 'any_associations_source', + 'bin_associations_binaries', + 'binaries_suite_arch', + 'changelogs', + 'file_arch_suite', + 'newest_all_associations', + 'newest_any_associations', + 'newest_source', + 'newest_src_association', + 'obsolete_all_associations', + 'obsolete_any_associations', + 'obsolete_any_by_all_associations', + 'obsolete_src_associations', + 'package_list', + 'source_suite', + 'src_associations_bin', + 'src_associations_src', + 'suite_arch_by_name', + ) + + for table_name in tables: + table = Table(table_name, self.db_meta, \ + autoload=True, useexisting=True) + setattr(self, 'tbl_%s' % table_name, table) + + for view_name in views: + view = Table(view_name, self.db_meta, autoload=True) + setattr(self, 'view_%s' % view_name, view) def __setupmappers(self): mapper(Architecture, self.tbl_architecture, - properties = dict(arch_id = self.tbl_architecture.c.id)) + properties = dict(arch_id = self.tbl_architecture.c.id, + suites = relation(Suite, secondary=self.tbl_suite_architectures, + order_by=self.tbl_suite.c.suite_name, + backref=backref('architectures', order_by=self.tbl_architecture.c.arch_string))), + extension = validator) + + mapper(ACL, self.tbl_acl, + properties = dict( + architectures = relation(Architecture, secondary=self.tbl_acl_architecture_map, collection_class=set), + fingerprints = relation(Fingerprint, secondary=self.tbl_acl_fingerprint_map, collection_class=set), + match_keyring = relation(Keyring, primaryjoin=(self.tbl_acl.c.match_keyring_id == self.tbl_keyrings.c.id)), + per_source = relation(ACLPerSource, collection_class=set), + )) + + mapper(ACLPerSource, self.tbl_acl_per_source, + properties = dict( + acl = relation(ACL), + fingerprint = relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.fingerprint_id == self.tbl_fingerprint.c.id)), + created_by = relation(Fingerprint, primaryjoin=(self.tbl_acl_per_source.c.created_by_id == self.tbl_fingerprint.c.id)), + )) mapper(Archive, self.tbl_archive, properties = dict(archive_id = self.tbl_archive.c.id, archive_name = self.tbl_archive.c.name)) - mapper(BinAssociation, self.tbl_bin_associations, - properties = dict(ba_id = self.tbl_bin_associations.c.id, - suite_id = self.tbl_bin_associations.c.suite, - suite = relation(Suite), - binary_id = self.tbl_bin_associations.c.bin, - binary = relation(DBBinary))) + mapper(ArchiveFile, self.tbl_files_archive_map, + properties = dict(archive = relation(Archive, backref='files'), + component = relation(Component), + file = relation(PoolFile, backref='archives'))) + mapper(BuildQueue, self.tbl_build_queue, + properties = dict(queue_id = self.tbl_build_queue.c.id, + suite = relation(Suite, primaryjoin=(self.tbl_build_queue.c.suite_id==self.tbl_suite.c.id)))) mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, @@ -2404,7 +2393,7 @@ class DBConn(Singleton): maintainer_id = self.tbl_binaries.c.maintainer, maintainer = relation(Maintainer), source_id = self.tbl_binaries.c.source, - source = relation(DBSource), + source = relation(DBSource, backref='binaries'), arch_id = self.tbl_binaries.c.architecture, architecture = relation(Architecture), poolfile_id = self.tbl_binaries.c.file, @@ -2413,20 +2402,18 @@ class DBConn(Singleton): fingerprint_id = self.tbl_binaries.c.sig_fpr, fingerprint = relation(Fingerprint), install_date = self.tbl_binaries.c.install_date, - binassociations = relation(BinAssociation, - primaryjoin=(self.tbl_binaries.c.id==self.tbl_bin_associations.c.bin)))) - - mapper(BinaryACL, self.tbl_binary_acl, - properties = dict(binary_acl_id = self.tbl_binary_acl.c.id)) - - mapper(BinaryACLMap, self.tbl_binary_acl_map, - properties = dict(binary_acl_map_id = self.tbl_binary_acl_map.c.id, - fingerprint = relation(Fingerprint, backref="binary_acl_map"), - architecture = relation(Architecture))) + suites = relation(Suite, secondary=self.tbl_bin_associations, + backref=backref('binaries', lazy='dynamic')), + extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references, + backref=backref('extra_binary_references', lazy='dynamic')), + key = relation(BinaryMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), + extension = validator) mapper(Component, self.tbl_component, properties = dict(component_id = self.tbl_component.c.id, - component_name = self.tbl_component.c.name)) + component_name = self.tbl_component.c.name), + extension = validator) mapper(DBConfig, self.tbl_config, properties = dict(config_id = self.tbl_config.c.id)) @@ -2438,11 +2425,17 @@ class DBConn(Singleton): poolfile_id = self.tbl_dsc_files.c.file, poolfile = relation(PoolFile))) + mapper(ExternalOverride, self.tbl_external_overrides, + properties = dict( + suite_id = self.tbl_external_overrides.c.suite, + suite = relation(Suite), + component_id = self.tbl_external_overrides.c.component, + component = relation(Component))) + mapper(PoolFile, self.tbl_files, properties = dict(file_id = self.tbl_files.c.id, - filesize = self.tbl_files.c.size, - location_id = self.tbl_files.c.location, - location = relation(Location))) + filesize = self.tbl_files.c.size), + extension = validator) mapper(Fingerprint, self.tbl_fingerprint, properties = dict(fingerprint_id = self.tbl_fingerprint.c.id, @@ -2450,160 +2443,245 @@ class DBConn(Singleton): uid = relation(Uid), keyring_id = self.tbl_fingerprint.c.keyring, keyring = relation(Keyring), - source_acl = relation(SourceACL), - binary_acl = relation(BinaryACL))) + acl = relation(ACL)), + extension = validator) mapper(Keyring, self.tbl_keyrings, properties = dict(keyring_name = self.tbl_keyrings.c.name, - keyring_id = self.tbl_keyrings.c.id)) - - mapper(KnownChange, self.tbl_known_changes, - properties = dict(known_change_id = self.tbl_known_changes.c.id)) - - mapper(KeyringACLMap, self.tbl_keyring_acl_map, - properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, - keyring = relation(Keyring, backref="keyring_acl_map"), - architecture = relation(Architecture))) - - mapper(Location, self.tbl_location, - properties = dict(location_id = self.tbl_location.c.id, - component_id = self.tbl_location.c.component, - component = relation(Component), - archive_id = self.tbl_location.c.archive, - archive = relation(Archive), - archive_type = self.tbl_location.c.type)) + keyring_id = self.tbl_keyrings.c.id, + acl = relation(ACL, primaryjoin=(self.tbl_keyrings.c.acl_id == self.tbl_acl.c.id)))), + + mapper(DBChange, self.tbl_changes, + properties = dict(change_id = self.tbl_changes.c.id, + seen = self.tbl_changes.c.seen, + source = self.tbl_changes.c.source, + binaries = self.tbl_changes.c.binaries, + architecture = self.tbl_changes.c.architecture, + distribution = self.tbl_changes.c.distribution, + urgency = self.tbl_changes.c.urgency, + maintainer = self.tbl_changes.c.maintainer, + changedby = self.tbl_changes.c.changedby, + date = self.tbl_changes.c.date, + version = self.tbl_changes.c.version)) mapper(Maintainer, self.tbl_maintainer, - properties = dict(maintainer_id = self.tbl_maintainer.c.id)) + properties = dict(maintainer_id = self.tbl_maintainer.c.id, + maintains_sources = relation(DBSource, backref='maintainer', + primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.maintainer)), + changed_sources = relation(DBSource, backref='changedby', + primaryjoin=(self.tbl_maintainer.c.id==self.tbl_source.c.changedby))), + extension = validator) mapper(NewComment, self.tbl_new_comments, - properties = dict(comment_id = self.tbl_new_comments.c.id)) + properties = dict(comment_id = self.tbl_new_comments.c.id, + policy_queue = relation(PolicyQueue))) mapper(Override, self.tbl_override, properties = dict(suite_id = self.tbl_override.c.suite, - suite = relation(Suite), + suite = relation(Suite, \ + backref=backref('overrides', lazy='dynamic')), + package = self.tbl_override.c.package, component_id = self.tbl_override.c.component, - component = relation(Component), + component = relation(Component, \ + backref=backref('overrides', lazy='dynamic')), priority_id = self.tbl_override.c.priority, - priority = relation(Priority), + priority = relation(Priority, \ + backref=backref('overrides', lazy='dynamic')), section_id = self.tbl_override.c.section, - section = relation(Section), + section = relation(Section, \ + backref=backref('overrides', lazy='dynamic')), overridetype_id = self.tbl_override.c.type, - overridetype = relation(OverrideType))) + overridetype = relation(OverrideType, \ + backref=backref('overrides', lazy='dynamic')))) mapper(OverrideType, self.tbl_override_type, properties = dict(overridetype = self.tbl_override_type.c.type, overridetype_id = self.tbl_override_type.c.id)) + mapper(PolicyQueue, self.tbl_policy_queue, + properties = dict(policy_queue_id = self.tbl_policy_queue.c.id, + suite = relation(Suite, primaryjoin=(self.tbl_policy_queue.c.suite_id == self.tbl_suite.c.id)))) + + mapper(PolicyQueueUpload, self.tbl_policy_queue_upload, + properties = dict( + changes = relation(DBChange), + policy_queue = relation(PolicyQueue, backref='uploads'), + target_suite = relation(Suite), + source = relation(DBSource), + binaries = relation(DBBinary, secondary=self.tbl_policy_queue_upload_binaries_map), + )) + + mapper(PolicyQueueByhandFile, self.tbl_policy_queue_byhand_file, + properties = dict( + upload = relation(PolicyQueueUpload, backref='byhand'), + ) + ) + mapper(Priority, self.tbl_priority, properties = dict(priority_id = self.tbl_priority.c.id)) - mapper(Queue, self.tbl_queue, - properties = dict(queue_id = self.tbl_queue.c.id)) - - mapper(QueueBuild, self.tbl_queue_build, - properties = dict(suite_id = self.tbl_queue_build.c.suite, - queue_id = self.tbl_queue_build.c.queue, - queue = relation(Queue, backref='queuebuild'))) - mapper(Section, self.tbl_section, - properties = dict(section_id = self.tbl_section.c.id)) + properties = dict(section_id = self.tbl_section.c.id, + section=self.tbl_section.c.section)) + + mapper(SignatureHistory, self.tbl_signature_history) mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, version = self.tbl_source.c.version, maintainer_id = self.tbl_source.c.maintainer, - maintainer = relation(Maintainer, - primaryjoin=(self.tbl_source.c.maintainer==self.tbl_maintainer.c.id)), poolfile_id = self.tbl_source.c.file, poolfile = relation(PoolFile), fingerprint_id = self.tbl_source.c.sig_fpr, fingerprint = relation(Fingerprint), changedby_id = self.tbl_source.c.changedby, - changedby = relation(Maintainer, - primaryjoin=(self.tbl_source.c.changedby==self.tbl_maintainer.c.id)), srcfiles = relation(DSCFile, primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)), - srcassociations = relation(SrcAssociation, - primaryjoin=(self.tbl_source.c.id==self.tbl_src_associations.c.source)), - srcuploaders = relation(SrcUploader))) - - mapper(SourceACL, self.tbl_source_acl, - properties = dict(source_acl_id = self.tbl_source_acl.c.id)) - - mapper(SrcAssociation, self.tbl_src_associations, - properties = dict(sa_id = self.tbl_src_associations.c.id, - suite_id = self.tbl_src_associations.c.suite, - suite = relation(Suite), - source_id = self.tbl_src_associations.c.source, - source = relation(DBSource))) + suites = relation(Suite, secondary=self.tbl_src_associations, + backref=backref('sources', lazy='dynamic')), + uploaders = relation(Maintainer, + secondary=self.tbl_src_uploaders), + key = relation(SourceMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), + extension = validator) mapper(SrcFormat, self.tbl_src_format, properties = dict(src_format_id = self.tbl_src_format.c.id, format_name = self.tbl_src_format.c.format_name)) - mapper(SrcUploader, self.tbl_src_uploaders, - properties = dict(uploader_id = self.tbl_src_uploaders.c.id, - source_id = self.tbl_src_uploaders.c.source, - source = relation(DBSource, - primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)), - maintainer_id = self.tbl_src_uploaders.c.maintainer, - maintainer = relation(Maintainer, - primaryjoin=(self.tbl_src_uploaders.c.maintainer==self.tbl_maintainer.c.id)))) - mapper(Suite, self.tbl_suite, - properties = dict(suite_id = self.tbl_suite.c.id)) - - mapper(SuiteArchitecture, self.tbl_suite_architectures, - properties = dict(suite_id = self.tbl_suite_architectures.c.suite, - suite = relation(Suite, backref='suitearchitectures'), - arch_id = self.tbl_suite_architectures.c.architecture, - architecture = relation(Architecture))) - - mapper(SuiteSrcFormat, self.tbl_suite_src_formats, - properties = dict(suite_id = self.tbl_suite_src_formats.c.suite, - suite = relation(Suite, backref='suitesrcformats'), - src_format_id = self.tbl_suite_src_formats.c.src_format, - src_format = relation(SrcFormat))) + properties = dict(suite_id = self.tbl_suite.c.id, + policy_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.policy_queue_id == self.tbl_policy_queue.c.id)), + new_queue = relation(PolicyQueue, primaryjoin=(self.tbl_suite.c.new_queue_id == self.tbl_policy_queue.c.id)), + debug_suite = relation(Suite, remote_side=[self.tbl_suite.c.id]), + copy_queues = relation(BuildQueue, + secondary=self.tbl_suite_build_queue_copy), + srcformats = relation(SrcFormat, secondary=self.tbl_suite_src_formats, + backref=backref('suites', lazy='dynamic')), + archive = relation(Archive, backref='suites'), + acls = relation(ACL, secondary=self.tbl_suite_acl_map, collection_class=set), + components = relation(Component, secondary=self.tbl_component_suite, + order_by=self.tbl_component.c.ordering, + backref=backref('suites'))), + extension = validator) mapper(Uid, self.tbl_uid, properties = dict(uid_id = self.tbl_uid.c.id, - fingerprint = relation(Fingerprint))) - - mapper(UploadBlock, self.tbl_upload_blocks, - properties = dict(upload_block_id = self.tbl_upload_blocks.c.id, - fingerprint = relation(Fingerprint, backref="uploadblocks"), - uid = relation(Uid, backref="uploadblocks"))) + fingerprint = relation(Fingerprint)), + extension = validator) + + mapper(BinContents, self.tbl_bin_contents, + properties = dict( + binary = relation(DBBinary, + backref=backref('contents', lazy='dynamic', cascade='all')), + file = self.tbl_bin_contents.c.file)) + + mapper(SrcContents, self.tbl_src_contents, + properties = dict( + source = relation(DBSource, + backref=backref('contents', lazy='dynamic', cascade='all')), + file = self.tbl_src_contents.c.file)) + + mapper(MetadataKey, self.tbl_metadata_keys, + properties = dict( + key_id = self.tbl_metadata_keys.c.key_id, + key = self.tbl_metadata_keys.c.key)) + + mapper(BinaryMetadata, self.tbl_binaries_metadata, + properties = dict( + binary_id = self.tbl_binaries_metadata.c.bin_id, + binary = relation(DBBinary), + key_id = self.tbl_binaries_metadata.c.key_id, + key = relation(MetadataKey), + value = self.tbl_binaries_metadata.c.value)) + + mapper(SourceMetadata, self.tbl_source_metadata, + properties = dict( + source_id = self.tbl_source_metadata.c.src_id, + source = relation(DBSource), + key_id = self.tbl_source_metadata.c.key_id, + key = relation(MetadataKey), + value = self.tbl_source_metadata.c.value)) + + mapper(VersionCheck, self.tbl_version_check, + properties = dict( + suite_id = self.tbl_version_check.c.suite, + suite = relation(Suite, primaryjoin=self.tbl_version_check.c.suite==self.tbl_suite.c.id), + reference_id = self.tbl_version_check.c.reference, + reference = relation(Suite, primaryjoin=self.tbl_version_check.c.reference==self.tbl_suite.c.id, lazy='joined'))) ## Connection functions def __createconn(self): from config import Config cnf = Config() - if cnf["DB::Host"]: + if cnf.has_key("DB::Service"): + connstr = "postgresql://service=%s" % cnf["DB::Service"] + elif cnf.has_key("DB::Host"): # TCP/IP - connstr = "postgres://%s" % cnf["DB::Host"] - if cnf["DB::Port"] and cnf["DB::Port"] != "-1": + connstr = "postgresql://%s" % cnf["DB::Host"] + if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": connstr += ":%s" % cnf["DB::Port"] connstr += "/%s" % cnf["DB::Name"] else: # Unix Socket - connstr = "postgres:///%s" % cnf["DB::Name"] - if cnf["DB::Port"] and cnf["DB::Port"] != "-1": + connstr = "postgresql:///%s" % cnf["DB::Name"] + if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": connstr += "?port=%s" % cnf["DB::Port"] - self.db_pg = create_engine(connstr, echo=self.debug) - self.db_meta = MetaData() - self.db_meta.bind = self.db_pg - self.db_smaker = sessionmaker(bind=self.db_pg, - autoflush=True, - autocommit=False) + engine_args = { 'echo': self.debug } + if cnf.has_key('DB::PoolSize'): + engine_args['pool_size'] = int(cnf['DB::PoolSize']) + if cnf.has_key('DB::MaxOverflow'): + engine_args['max_overflow'] = int(cnf['DB::MaxOverflow']) + if sa_major_version != '0.5' and cnf.has_key('DB::Unicode') and \ + cnf['DB::Unicode'] == 'false': + engine_args['use_native_unicode'] = False + + # Monkey patch a new dialect in in order to support service= syntax + import sqlalchemy.dialects.postgresql + from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 + class PGDialect_psycopg2_dak(PGDialect_psycopg2): + def create_connect_args(self, url): + if str(url).startswith('postgresql://service='): + # Eww + servicename = str(url)[21:] + return (['service=%s' % servicename], {}) + else: + return PGDialect_psycopg2.create_connect_args(self, url) - self.__setuptables() - self.__setupmappers() + sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak - def session(self): - return self.db_smaker() + try: + self.db_pg = create_engine(connstr, **engine_args) + self.db_meta = MetaData() + self.db_meta.bind = self.db_pg + self.db_smaker = sessionmaker(bind=self.db_pg, + autoflush=True, + autocommit=False) + + self.__setuptables() + self.__setupmappers() + + except OperationalError as e: + import utils + utils.fubar("Cannot connect to database (%s)" % str(e)) + + self.pid = os.getpid() + + def session(self, work_mem = 0): + ''' + Returns a new session object. If a work_mem parameter is provided a new + transaction is started and the work_mem parameter is set for this + transaction. The work_mem parameter is measured in MB. A default value + will be used if the parameter is not set. + ''' + # reinitialize DBConn in new processes + if self.pid != os.getpid(): + clear_mappers() + self.__createconn() + session = self.db_smaker() + if work_mem > 0: + session.execute("SET LOCAL work_mem TO '%d MB'" % work_mem) + return session __all__.append('DBConn') - -