################################################################################
import os
+from os.path import normpath
import re
import psycopg2
import traceback
from datetime import datetime, timedelta
from errno import ENOENT
from tempfile import mkstemp, mkdtemp
+from subprocess import Popen, PIPE
+from tarfile import TarFile
from inspect import getargspec
import sqlalchemy
-from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc
+from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \
+ Text, ForeignKey
from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \
- backref, MapperExtension, EXT_CONTINUE
+ backref, MapperExtension, EXT_CONTINUE, object_mapper
from sqlalchemy import types as sqltypes
# Don't remove this, we re-export the exceptions to scripts which import us
'''
return session.query(cls).get(primary_key)
+ def session(self, replace = False):
+ '''
+ Returns the current session that is associated with the object. May
+ return None is object is in detached state.
+ '''
+
+ return object_session(self)
+
+ def clone(self, session = None):
+ '''
+ Clones the current object in a new session and returns the new clone. A
+ fresh session is created if the optional session parameter is not
+ provided. The function will fail if a session is provided and has
+ unflushed changes.
+
+ RATIONALE: SQLAlchemy's session is not thread safe. This method clones
+ an existing object to allow several threads to work with their own
+ instances of an ORMObject.
+
+ WARNING: Only persistent (committed) objects can be cloned. Changes
+ made to the original object that are not committed yet will get lost.
+ The session of the new object will always be rolled back to avoid
+ ressource leaks.
+ '''
+
+ if self.session() is None:
+ raise RuntimeError( \
+ 'Method clone() failed for detached object:\n%s' % self)
+ self.session().flush()
+ mapper = object_mapper(self)
+ primary_key = mapper.primary_key_from_instance(self)
+ object_class = self.__class__
+ if session is None:
+ session = DBConn().session()
+ elif len(session.new) + len(session.dirty) + len(session.deleted) > 0:
+ raise RuntimeError( \
+ 'Method clone() failed due to unflushed changes in session.')
+ new_object = session.query(object_class).get(primary_key)
+ session.rollback()
+ if new_object is None:
+ raise RuntimeError( \
+ 'Method clone() failed for non-persistent object:\n%s' % self)
+ return new_object
+
__all__.append('ORMObject')
################################################################################
################################################################################
-class BinContents(object):
- def __init__(self, *args, **kwargs):
- pass
+class BinContents(ORMObject):
+ def __init__(self, file = None, binary = None):
+ self.file = file
+ self.binary = binary
- def __repr__(self):
- return '<BinContents (%s, %s)>' % (self.binary, self.filename)
+ def properties(self):
+ return ['file', 'binary']
__all__.append('BinContents')
def properties(self):
return ['package', 'version', 'maintainer', 'source', 'architecture', \
'poolfile', 'binarytype', 'fingerprint', 'install_date', \
- 'suites_count', 'binary_id']
+ 'suites_count', 'binary_id', 'contents_count']
def not_null_constraints(self):
return ['package', 'version', 'maintainer', 'source', 'poolfile', \
def get_component_name(self):
return self.poolfile.location.component.component_name
+ def scan_contents(self):
+ '''
+ Yields the contents of the package. Only regular files are yielded and
+ the path names are normalized after converting them from either utf-8
+ or iso8859-1 encoding. It yields the string ' <EMPTY PACKAGE>' if the
+ package does not contain any regular file.
+ '''
+ fullpath = self.poolfile.fullpath
+ dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE)
+ tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|')
+ for member in tar.getmembers():
+ if not member.isdir():
+ name = normpath(member.name)
+ # enforce proper utf-8 encoding
+ try:
+ name.decode('utf-8')
+ except UnicodeDecodeError:
+ name = name.decode('iso8859-1').encode('utf-8')
+ yield name
+ tar.close()
+ dpkg.stdout.close()
+ dpkg.wait()
+
__all__.append('DBBinary')
@session_wrapper
return NotImplemented
def properties(self):
- return ['component_name', 'component_id', 'description', 'location', \
- 'meets_dfsg']
+ return ['component_name', 'component_id', 'description', \
+ 'location_count', 'meets_dfsg', 'overrides_count']
def not_null_constraints(self):
return ['component_name']
def fullpath(self):
return os.path.join(self.location.path, self.filename)
- def is_valid(self, filesize = -1, md5sum = None):\
- return self.filesize == filesize and self.md5sum == md5sum
+ def is_valid(self, filesize = -1, md5sum = None):
+ return self.filesize == long(filesize) and self.md5sum == md5sum
def properties(self):
return ['filename', 'file_id', 'filesize', 'md5sum', 'sha1sum', \
self.archive_type = 'pool'
def properties(self):
- return ['path', 'archive_type', 'component', 'files_count']
+ return ['path', 'location_id', 'archive_type', 'component', \
+ 'files_count']
def not_null_constraints(self):
return ['path', 'archive_type']
################################################################################
-class Override(object):
- def __init__(self, *args, **kwargs):
- pass
+class Override(ORMObject):
+ def __init__(self, package = None, suite = None, component = None, overridetype = None, \
+ section = None, priority = None):
+ self.package = package
+ self.suite = suite
+ self.component = component
+ self.overridetype = overridetype
+ self.section = section
+ self.priority = priority
- def __repr__(self):
- return '<Override %s (%s)>' % (self.package, self.suite_id)
+ def properties(self):
+ return ['package', 'suite', 'component', 'overridetype', 'section', \
+ 'priority']
+
+ def not_null_constraints(self):
+ return ['package', 'suite', 'component', 'overridetype', 'section']
__all__.append('Override')
################################################################################
-class OverrideType(object):
- def __init__(self, *args, **kwargs):
- pass
+class OverrideType(ORMObject):
+ def __init__(self, overridetype = None):
+ self.overridetype = overridetype
- def __repr__(self):
- return '<OverrideType %s>' % self.overridetype
+ def properties(self):
+ return ['overridetype', 'overridetype_id', 'overrides_count']
+
+ def not_null_constraints(self):
+ return ['overridetype']
__all__.append('OverrideType')
################################################################################
-class Priority(object):
- def __init__(self, *args, **kwargs):
- pass
+class Priority(ORMObject):
+ def __init__(self, priority = None, level = None):
+ self.priority = priority
+ self.level = level
+
+ def properties(self):
+ return ['priority', 'priority_id', 'level', 'overrides_count']
+
+ def not_null_constraints(self):
+ return ['priority', 'level']
def __eq__(self, val):
if isinstance(val, str):
# This signals to use the normal comparison operator
return NotImplemented
- def __repr__(self):
- return '<Priority %s (%s)>' % (self.priority, self.priority_id)
-
__all__.append('Priority')
@session_wrapper
################################################################################
-class Section(object):
- def __init__(self, *args, **kwargs):
- pass
+class Section(ORMObject):
+ def __init__(self, section = None):
+ self.section = section
+
+ def properties(self):
+ return ['section', 'section_id', 'overrides_count']
+
+ def not_null_constraints(self):
+ return ['section']
def __eq__(self, val):
if isinstance(val, str):
# This signals to use the normal comparison operator
return NotImplemented
- def __repr__(self):
- return '<Section %s>' % self.section
-
__all__.append('Section')
@session_wrapper
maps = [ (x[1], x[2]) for x in maps
if x[0] == "map" or x[0] == "silent-map" ]
s = [suite]
- for x in maps:
- if x[1] in s and x[0] not in s:
- s.append(x[0])
+ for (from_, to) in maps:
+ if from_ in s and to not in s:
+ s.append(to)
q = q.filter(DBSource.suites.any(Suite.suite_name.in_(s)))
self.version = version
def properties(self):
- return ['suite_name', 'version', 'sources_count', 'binaries_count']
+ return ['suite_name', 'version', 'sources_count', 'binaries_count', \
+ 'overrides_count']
def not_null_constraints(self):
return ['suite_name', 'version']
'binary_acl',
'binary_acl_map',
'build_queue',
+ 'build_queue_files',
'changelogs_text',
+ 'changes',
'component',
'config',
'changes_pending_binaries',
'suite',
'uid',
'upload_blocks',
- # The following tables have primary keys but sqlalchemy
- # version 0.5 fails to reflect them correctly with database
- # versions before upgrade #41.
- #'changes',
- #'build_queue_files',
)
tables_no_primary = (
- 'bin_contents',
'changes_pending_files_map',
'changes_pending_source_files',
'changes_pool_files',
'deb_contents',
+ # TODO: the maintainer column in table override should be removed.
'override',
'suite_architectures',
'suite_src_formats',
'suite_build_queue_copy',
'udeb_contents',
- # see the comment above
- 'changes',
- 'build_queue_files',
)
views = (
table = Table(table_name, self.db_meta, autoload=True)
setattr(self, 'tbl_%s' % table_name, table)
+ # bin_contents needs special attention until update #41 has been
+ # applied
+ self.tbl_bin_contents = Table('bin_contents', self.db_meta, \
+ Column('file', Text, primary_key = True),
+ Column('binary_id', Integer, ForeignKey('binaries.id'), \
+ primary_key = True),
+ autoload=True, useexisting=True)
+
for view_name in views:
view = Table(view_name, self.db_meta, autoload=True)
setattr(self, 'view_%s' % view_name, view)
mapper(Location, self.tbl_location,
properties = dict(location_id = self.tbl_location.c.id,
component_id = self.tbl_location.c.component,
- component = relation(Component, \
- backref=backref('location', uselist = False)),
+ component = relation(Component, backref='location'),
archive_id = self.tbl_location.c.archive,
archive = relation(Archive),
# FIXME: the 'type' column is old cruft and
mapper(Override, self.tbl_override,
properties = dict(suite_id = self.tbl_override.c.suite,
- suite = relation(Suite),
+ suite = relation(Suite, \
+ backref=backref('overrides', lazy='dynamic')),
package = self.tbl_override.c.package,
component_id = self.tbl_override.c.component,
- component = relation(Component),
+ component = relation(Component, \
+ backref=backref('overrides', lazy='dynamic')),
priority_id = self.tbl_override.c.priority,
- priority = relation(Priority),
+ priority = relation(Priority, \
+ backref=backref('overrides', lazy='dynamic')),
section_id = self.tbl_override.c.section,
- section = relation(Section),
+ section = relation(Section, \
+ backref=backref('overrides', lazy='dynamic')),
overridetype_id = self.tbl_override.c.type,
- overridetype = relation(OverrideType)))
+ overridetype = relation(OverrideType, \
+ backref=backref('overrides', lazy='dynamic'))))
mapper(OverrideType, self.tbl_override_type,
properties = dict(overridetype = self.tbl_override_type.c.type,
fingerprint = relation(Fingerprint, backref="uploadblocks"),
uid = relation(Uid, backref="uploadblocks")))
+ mapper(BinContents, self.tbl_bin_contents,
+ properties = dict(
+ binary = relation(DBBinary,
+ backref=backref('contents', lazy='dynamic', cascade='all')),
+ file = self.tbl_bin_contents.c.file))
+
## Connection functions
def __createconn(self):
from config import Config
if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
connstr += "?port=%s" % cnf["DB::Port"]
- self.db_pg = create_engine(connstr, echo=self.debug)
+ engine_args = { 'echo': self.debug }
+ if cnf.has_key('DB::PoolSize'):
+ engine_args['pool_size'] = int(cnf['DB::PoolSize'])
+ if cnf.has_key('DB::MaxOverflow'):
+ engine_args['max_overflow'] = int(cnf['DB::MaxOverflow'])
+ if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \
+ cnf['DB::Unicode'] == 'false':
+ engine_args['use_native_unicode'] = False
+
+ self.db_pg = create_engine(connstr, **engine_args)
self.db_meta = MetaData()
self.db_meta.bind = self.db_pg
self.db_smaker = sessionmaker(bind=self.db_pg,
projects / dak.git / blobdiff