X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=4d30e663b994fee443f87aa120dbca0fb9bc1a03;hb=b8690d6b1256a4a95198b874d9eb1aab62a825fb;hp=1bf5a0f997c7bb84aa5821aa4f9ab3c074ec3422;hpb=144898960af3c564c0a93203cafff457636d2163;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 1bf5a0f9..4d30e663 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -59,8 +59,10 @@ import sqlalchemy from sqlalchemy import create_engine, Table, MetaData, Column, Integer, desc, \ Text, ForeignKey from sqlalchemy.orm import sessionmaker, mapper, relation, object_session, \ - backref, MapperExtension, EXT_CONTINUE, object_mapper + backref, MapperExtension, EXT_CONTINUE, object_mapper, clear_mappers from sqlalchemy import types as sqltypes +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.ext.associationproxy import association_proxy # Don't remove this, we re-export the exceptions to scripts which import us from sqlalchemy.exc import * @@ -493,12 +495,14 @@ class DBBinary(ORMObject): def properties(self): return ['package', 'version', 'maintainer', 'source', 'architecture', \ 'poolfile', 'binarytype', 'fingerprint', 'install_date', \ - 'suites_count', 'binary_id', 'contents_count'] + 'suites_count', 'binary_id', 'contents_count', 'extra_sources'] def not_null_constraints(self): return ['package', 'version', 'maintainer', 'source', 'poolfile', \ 'binarytype'] + metadata = association_proxy('key', 'value') + def get_component_name(self): return self.poolfile.location.component.component_name @@ -525,6 +529,25 @@ class DBBinary(ORMObject): dpkg.stdout.close() dpkg.wait() + def read_control(self): + ''' + Reads the control information from a binary. + + @rtype: tuple + @return: (stanza, controldict) stanza is the text of the control + section. controldict is the information in a dictionary + form + ''' + import apt_inst, apt_pk + fullpath = self.poolfile.fullpath + deb_file = open(fullpath, 'r') + stanza = apt_inst.debExtractControl(deb_file).rstrip() + control = dict(apt_pkg.TagSection(stanza)) + deb_file.close() + + return stanza, control + + __all__.append('DBBinary') @session_wrapper @@ -2172,6 +2195,22 @@ class DBSource(ORMObject): return ['source', 'version', 'install_date', 'maintainer', \ 'changedby', 'poolfile', 'install_date'] + def read_control(self): + ''' + Reads the control information from a dsc + + @rtype: tuple + @return: (stanza, controldict) stanza is the text of the control + section. controldict is the information in a dictionary + form + ''' + from debian.debfile import Deb822 + fullpath = self.poolfile.fullpath + fields = Deb822(open(self.poolfile.fullpath, 'r')) + return fields + + metadata = association_proxy('key', 'value') + __all__.append('DBSource') @session_wrapper @@ -2465,6 +2504,16 @@ def add_deb_to_db(u, filename, session=None): bin.source_id = bin_sources[0].source_id + if entry.has_key("built-using"): + for srcname, version in entry["built-using"]: + exsources = get_sources_from_name(srcname, version, session=session) + if len(exsources) != 1: + raise NoSourceFieldError, "Unable to find source package (%s = %s) in Built-Using for %s (%s), %s, file %s, type %s, signed by %s" % \ + (srcname, version, bin.package, bin.version, entry["architecture"], + filename, bin.binarytype, u.pkg.changes["fingerprint"]) + + bin.extra_sources.append(exsources[0]) + # Add and flush object so it has an ID session.add(bin) @@ -2792,6 +2841,52 @@ __all__.append('UploadBlock') ################################################################################ +class MetadataKey(ORMObject): + def __init__(self, key = None): + self.key = key + + def properties(self): + return ['key'] + + def not_null_constraints(self): + return ['key'] + +__all__.append('MetadataKey') + +################################################################################ + +class BinaryMetadata(ORMObject): + def __init__(self, key = None, value = None, binary = None): + self.key = key + self.value = value + self.binary = binary + + def properties(self): + return ['binary', 'key', 'value'] + + def not_null_constraints(self): + return ['value'] + +__all__.append('BinaryMetadata') + +################################################################################ + +class SourceMetadata(ORMObject): + def __init__(self, key = None, value = None, source = None): + self.key = key + self.value = value + self.source = source + + def properties(self): + return ['source', 'key', 'value'] + + def not_null_constraints(self): + return ['value'] + +__all__.append('SourceMetadata') + +################################################################################ + class DBConn(object): """ database module init. @@ -2807,11 +2902,13 @@ class DBConn(object): self.__createconn() def __setuptables(self): - tables_with_primary = ( + tables = ( 'architecture', 'archive', 'bin_associations', + 'bin_contents', 'binaries', + 'binaries_metadata', 'binary_acl', 'binary_acl_map', 'build_queue', @@ -2823,37 +2920,37 @@ class DBConn(object): 'changes_pending_binaries', 'changes_pending_files', 'changes_pending_source', + 'changes_pending_files_map', + 'changes_pending_source_files', + 'changes_pool_files', 'dsc_files', + 'extra_src_references', 'files', 'fingerprint', 'keyrings', 'keyring_acl_map', 'location', 'maintainer', + 'metadata_keys', 'new_comments', + # TODO: the maintainer column in table override should be removed. + 'override', 'override_type', 'policy_queue', 'priority', 'section', 'source', 'source_acl', + 'source_metadata', 'src_associations', 'src_format', 'src_uploaders', 'suite', - 'uid', - 'upload_blocks', - ) - - tables_no_primary = ( - 'changes_pending_files_map', - 'changes_pending_source_files', - 'changes_pool_files', - # TODO: the maintainer column in table override should be removed. - 'override', 'suite_architectures', - 'suite_src_formats', 'suite_build_queue_copy', + 'suite_src_formats', + 'uid', + 'upload_blocks', ) views = ( @@ -2880,28 +2977,11 @@ class DBConn(object): 'suite_arch_by_name', ) - # Sqlalchemy version 0.5 fails to reflect the SERIAL type - # correctly and that is why we have to use a workaround. It can - # be removed as soon as we switch to version 0.6. - for table_name in tables_with_primary: + for table_name in tables: table = Table(table_name, self.db_meta, \ - Column('id', Integer, primary_key = True), \ autoload=True, useexisting=True) setattr(self, 'tbl_%s' % table_name, table) - for table_name in tables_no_primary: - table = Table(table_name, self.db_meta, autoload=True) - setattr(self, 'tbl_%s' % table_name, table) - - # bin_contents needs special attention until the SERIAL type is - # correctly detected and the workaround has been removed; see comment - # above - self.tbl_bin_contents = Table('bin_contents', self.db_meta, \ - Column('file', Text, primary_key = True), - Column('binary_id', Integer, ForeignKey('binaries.id'), \ - primary_key = True), - autoload=True, useexisting=True) - for view_name in views: view = Table(view_name, self.db_meta, autoload=True) setattr(self, 'view_%s' % view_name, view) @@ -2942,7 +3022,11 @@ class DBConn(object): fingerprint = relation(Fingerprint), install_date = self.tbl_binaries.c.install_date, suites = relation(Suite, secondary=self.tbl_bin_associations, - backref=backref('binaries', lazy='dynamic'))), + backref=backref('binaries', lazy='dynamic')), + extra_sources = relation(DBSource, secondary=self.tbl_extra_src_references, + backref=backref('extra_binary_references', lazy='dynamic')), + key = relation(BinaryMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), extension = validator) mapper(BinaryACL, self.tbl_binary_acl, @@ -3112,7 +3196,9 @@ class DBConn(object): primaryjoin=(self.tbl_source.c.id==self.tbl_dsc_files.c.source)), suites = relation(Suite, secondary=self.tbl_src_associations, backref=backref('sources', lazy='dynamic')), - srcuploaders = relation(SrcUploader)), + srcuploaders = relation(SrcUploader), + key = relation(SourceMetadata, cascade='all', + collection_class=attribute_mapped_collection('key'))), extension = validator) mapper(SourceACL, self.tbl_source_acl, @@ -3160,20 +3246,43 @@ class DBConn(object): backref=backref('contents', lazy='dynamic', cascade='all')), file = self.tbl_bin_contents.c.file)) + mapper(MetadataKey, self.tbl_metadata_keys, + properties = dict( + key_id = self.tbl_metadata_keys.c.key_id, + key = self.tbl_metadata_keys.c.key)) + + mapper(BinaryMetadata, self.tbl_binaries_metadata, + properties = dict( + binary_id = self.tbl_binaries_metadata.c.bin_id, + binary = relation(DBBinary), + key_id = self.tbl_binaries_metadata.c.key_id, + key = relation(MetadataKey), + value = self.tbl_binaries_metadata.c.value)) + + mapper(SourceMetadata, self.tbl_source_metadata, + properties = dict( + source_id = self.tbl_source_metadata.c.src_id, + source = relation(DBSource), + key_id = self.tbl_source_metadata.c.key_id, + key = relation(MetadataKey), + value = self.tbl_source_metadata.c.value)) + ## Connection functions def __createconn(self): from config import Config cnf = Config() - if cnf["DB::Host"]: + if cnf.has_key("DB::Service"): + connstr = "postgresql://service=%s" % cnf["DB::Service"] + elif cnf.has_key("DB::Host"): # TCP/IP - connstr = "postgres://%s" % cnf["DB::Host"] - if cnf["DB::Port"] and cnf["DB::Port"] != "-1": + connstr = "postgresql://%s" % cnf["DB::Host"] + if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": connstr += ":%s" % cnf["DB::Port"] connstr += "/%s" % cnf["DB::Name"] else: # Unix Socket - connstr = "postgres:///%s" % cnf["DB::Name"] - if cnf["DB::Port"] and cnf["DB::Port"] != "-1": + connstr = "postgresql:///%s" % cnf["DB::Name"] + if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": connstr += "?port=%s" % cnf["DB::Port"] engine_args = { 'echo': self.debug } @@ -3185,6 +3294,20 @@ class DBConn(object): cnf['DB::Unicode'] == 'false': engine_args['use_native_unicode'] = False + # Monkey patch a new dialect in in order to support service= syntax + import sqlalchemy.dialects.postgresql + from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 + class PGDialect_psycopg2_dak(PGDialect_psycopg2): + def create_connect_args(self, url): + if str(url).startswith('postgresql://service='): + # Eww + servicename = str(url)[21:] + return (['service=%s' % servicename], {}) + else: + return PGDialect_psycopg2.create_connect_args(self, url) + + sqlalchemy.dialects.postgresql.base.dialect = PGDialect_psycopg2_dak + self.db_pg = create_engine(connstr, **engine_args) self.db_meta = MetaData() self.db_meta.bind = self.db_pg @@ -3194,8 +3317,13 @@ class DBConn(object): self.__setuptables() self.__setupmappers() + self.pid = os.getpid() def session(self): + # reinitialize DBConn in new processes + if self.pid != os.getpid(): + clear_mappers() + self.__createconn() return self.db_smaker() __all__.append('DBConn')