X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=6cd84de3f30da9fe9c93167c9954b17929dbc8d1;hb=a881423355b3e9455f44bb88d806bc04a7c7e7e5;hp=1bf44c858650951be219bfdeb1a5b8c360709dec;hpb=3fac3e435be2b886f6db5f2ef55d12ed3a32cac6;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 1bf44c85..6cd84de3 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -59,7 +59,7 @@ import sqlalchemy from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \ Text, ForeignKey from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \ - backref, MapperExtension, EXT_CONTINUE, object_mapper + backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers from sqlalchemy import types as sqltypes # Don't remove this, we re-export the exceptions to scripts which import us @@ -77,9 +77,6 @@ import warnings warnings.filterwarnings('ignore', \ "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \ SADeprecationWarning) -# TODO: sqlalchemy needs some extra configuration to correctly reflect -# the ind_deb_contents_* indexes - we ignore the warnings at the moment -warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning) ################################################################################ @@ -508,19 +505,22 @@ class DBBinary(ORMObject): def scan_contents(self): ''' Yields the contents of the package. Only regular files are yielded and - the path names are normalized after converting them from either utf-8 or - iso8859-1 encoding. + the path names are normalized after converting them from either utf-8 + or iso8859-1 encoding. It yields the string ' ' if the + package does not contain any regular file. ''' fullpath = self.poolfile.fullpath dpkg = Popen(['dpkg-deb', '--fsys-tarfile', fullpath], stdout = PIPE) tar = TarFile.open(fileobj = dpkg.stdout, mode = 'r|') for member in tar.getmembers(): - if member.isfile(): + if not member.isdir(): + name = normpath(member.name) + # enforce proper utf-8 encoding try: - name = member.name.decode('utf-8') + name.decode('utf-8') except UnicodeDecodeError: - name = member.name.decode('iso8859-1') - yield normpath(name) + name = name.decode('iso8859-1').encode('utf-8') + yield name tar.close() dpkg.stdout.close() dpkg.wait() @@ -1945,111 +1945,6 @@ __all__.append('get_override_type') ################################################################################ -class DebContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.package.package,self.file) - -__all__.append('DebContents') - - -class UdebContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % (self.package.package,self.file) - -__all__.append('UdebContents') - -class PendingBinContents(object): - def __init__(self, *args, **kwargs): - pass - - def __repr__(self): - return '' % self.contents_id - -__all__.append('PendingBinContents') - -def insert_pending_content_paths(package, - is_udeb, - fullpaths, - session=None): - """ - Make sure given paths are temporarily associated with given - package - - @type package: dict - @param package: the package to associate with should have been read in from the binary control file - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the caller - is responsible for ensuring a transaction has begun and committing the - results or rolling back based on the result code. If not passed, a commit - will be performed at the end of the function - - @return: True upon success, False if there is a problem - """ - - privatetrans = False - - if session is None: - session = DBConn().session() - privatetrans = True - - try: - arch = get_architecture(package['Architecture'], session) - arch_id = arch.arch_id - - # Remove any already existing recorded files for this package - q = session.query(PendingBinContents) - q = q.filter_by(package=package['Package']) - q = q.filter_by(version=package['Version']) - q = q.filter_by(architecture=arch_id) - q.delete() - - for fullpath in fullpaths: - - if fullpath.startswith( "./" ): - fullpath = fullpath[2:] - - pca = PendingBinContents() - pca.package = package['Package'] - pca.version = package['Version'] - pca.file = fullpath - pca.architecture = arch_id - - if isudeb: - pca.type = 8 # gross - else: - pca.type = 7 # also gross - session.add(pca) - - # Only commit if we set up the session ourself - if privatetrans: - session.commit() - session.close() - else: - session.flush() - - return True - except Exception, e: - traceback.print_exc() - - # Only rollback if we set up the session ourself - if privatetrans: - session.rollback() - session.close() - - return False - -__all__.append('insert_pending_content_paths') - -################################################################################ - class PolicyQueue(object): def __init__(self, *args, **kwargs): pass @@ -2937,7 +2832,6 @@ class DBConn(object): 'maintainer', 'new_comments', 'override_type', - 'pending_bin_contents', 'policy_queue', 'priority', 'section', @@ -2955,13 +2849,11 @@ class DBConn(object): 'changes_pending_files_map', 'changes_pending_source_files', 'changes_pool_files', - 'deb_contents', # TODO: the maintainer column in table override should be removed. 'override', 'suite_architectures', 'suite_src_formats', 'suite_build_queue_copy', - 'udeb_contents', ) views = ( @@ -3001,8 +2893,9 @@ class DBConn(object): table = Table(table_name, self.db_meta, autoload=True) setattr(self, 'tbl_%s' % table_name, table) - # bin_contents needs special attention until update #41 has been - # applied + # bin_contents needs special attention until the SERIAL type is + # correctly detected and the workaround has been removed; see comment + # above self.tbl_bin_contents = Table('bin_contents', self.db_meta, \ Column('file', Text, primary_key = True), Column('binary_id', Integer, ForeignKey('binaries.id'), \ @@ -3025,30 +2918,6 @@ class DBConn(object): properties = dict(archive_id = self.tbl_archive.c.id, archive_name = self.tbl_archive.c.name)) - mapper(PendingBinContents, self.tbl_pending_bin_contents, - properties = dict(contents_id =self.tbl_pending_bin_contents.c.id, - filename = self.tbl_pending_bin_contents.c.filename, - package = self.tbl_pending_bin_contents.c.package, - version = self.tbl_pending_bin_contents.c.version, - arch = self.tbl_pending_bin_contents.c.arch, - otype = self.tbl_pending_bin_contents.c.type)) - - mapper(DebContents, self.tbl_deb_contents, - properties = dict(binary_id=self.tbl_deb_contents.c.binary_id, - package=self.tbl_deb_contents.c.package, - suite=self.tbl_deb_contents.c.suite, - arch=self.tbl_deb_contents.c.arch, - section=self.tbl_deb_contents.c.section, - filename=self.tbl_deb_contents.c.filename)) - - mapper(UdebContents, self.tbl_udeb_contents, - properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id, - package=self.tbl_udeb_contents.c.package, - suite=self.tbl_udeb_contents.c.suite, - arch=self.tbl_udeb_contents.c.arch, - section=self.tbl_udeb_contents.c.section, - filename=self.tbl_udeb_contents.c.filename)) - mapper(BuildQueue, self.tbl_build_queue, properties = dict(queue_id = self.tbl_build_queue.c.id)) @@ -3288,7 +3157,7 @@ class DBConn(object): mapper(BinContents, self.tbl_bin_contents, properties = dict( binary = relation(DBBinary, - backref=backref('contents', lazy='dynamic')), + backref=backref('contents', lazy='dynamic', cascade='all')), file = self.tbl_bin_contents.c.file)) ## Connection functions @@ -3306,14 +3175,17 @@ class DBConn(object): connstr = "postgres:///%s" % cnf["DB::Name"] if cnf["DB::Port"] and cnf["DB::Port"] != "-1": connstr += "?port=%s" % cnf["DB::Port"] - if not cnf.has_key('DB::PoolSize'): - cnf['DB::PoolSize'] = '5' - if not cnf.has_key('DB::MaxOverflow'): - cnf['DB::MaxOverflow'] = '10' - - self.db_pg = create_engine(connstr, echo=self.debug, - pool_size=int(cnf['DB::PoolSize']), - max_overflow=int(cnf['DB::MaxOverflow'])) + + engine_args = { 'echo': self.debug } + if cnf.has_key('DB::PoolSize'): + engine_args['pool_size'] = int(cnf['DB::PoolSize']) + if cnf.has_key('DB::MaxOverflow'): + engine_args['max_overflow'] = int(cnf['DB::MaxOverflow']) + if sa_major_version == '0.6' and cnf.has_key('DB::Unicode') and \ + cnf['DB::Unicode'] == 'false': + engine_args['use_native_unicode'] = False + + self.db_pg = create_engine(connstr, **engine_args) self.db_meta = MetaData() self.db_meta.bind = self.db_pg self.db_smaker = sessionmaker(bind=self.db_pg, @@ -3326,6 +3198,15 @@ class DBConn(object): def session(self): return self.db_smaker() + def reset(self): + ''' + Resets the DBConn object. This function must be called by subprocesses + created by the multiprocessing module. See tests/dbtest_multiproc.py + for an example. + ''' + clear_mappers() + self.__createconn() + __all__.append('DBConn')