X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=42ef7e9f78eb5da11fd116ed88231fba25791fd5;hb=878b2ab2cacab6722e4bfce9d47988fe46e3b088;hp=23fff4f4ffaf3061c4750b8d9eba815490b326e5;hpb=129f300c804fb319e0e9970bba9d54bec5619025;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 23fff4f4..42ef7e9f 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -40,24 +40,14 @@ import traceback from sqlalchemy import create_engine, Table, MetaData, select from sqlalchemy.orm import sessionmaker, mapper, relation +# Don't remove this, we re-export the exceptions to scripts which import us +from sqlalchemy.exc import * + from singleton import Singleton ################################################################################ -class Cache(object): - def __init__(self, hashfunc=None): - if hashfunc: - self.hashfunc = hashfunc - else: - self.hashfunc = lambda x: str(x) - - self.data = {} - - def SetValue(self, keys, value): - self.data[self.hashfunc(keys)] = value - - def GetValue(self, keys): - return self.data.get(self.hashfunc(keys)) +__all__ = ['IntegrityError', 'SQLAlchemyError'] ################################################################################ @@ -68,6 +58,59 @@ class Architecture(object): def __repr__(self): return '' % self.arch_string +__all__.append('Architecture') + +def get_architecture(architecture, session=None): + """ + Returns database id for given C{architecture}. + + @type architecture: string + @param architecture: The name of the architecture + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Architecture + @return: Architecture object for the given arch (None if not present) + + """ + if session is None: + session = DBConn().session() + q = session.query(Architecture).filter_by(arch_string=architecture) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_architecture') + +def get_architecture_suites(architecture, session=None): + """ + Returns list of Suite objects for given C{architecture} name + + @type source: str + @param source: Architecture name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: list of Suite objects for the given name (may be empty) + """ + + if session is None: + session = DBConn().session() + + q = session.query(Suite) + q = q.join(SuiteArchitecture) + q = q.join(Architecture).filter_by(arch_string=architecture).order_by('suite_name') + return q.all() + +__all__.append('get_architecture_suites') + +################################################################################ + class Archive(object): def __init__(self, *args, **kwargs): pass @@ -75,6 +118,35 @@ class Archive(object): def __repr__(self): return '' % self.name +__all__.append('Archive') + +def get_archive(archive, session=None): + """ + returns database id for given c{archive}. + + @type archive: string + @param archive: the name of the arhive + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Archive + @return: Archive object for the given name (None if not present) + + """ + archive = archive.lower() + if session is None: + session = DBConn().session() + q = session.query(Archive).filter_by(archive_name=archive) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_archive') + +################################################################################ + class BinAssociation(object): def __init__(self, *args, **kwargs): pass @@ -82,15 +154,63 @@ class BinAssociation(object): def __repr__(self): return '' % (self.ba_id, self.binary, self.suite) -class Binary(object): +__all__.append('BinAssociation') + +################################################################################ + +class DBBinary(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % (self.package, self.version, self.architecture) + return '' % (self.package, self.version, self.architecture) + +__all__.append('DBBinary') + +def get_binary_from_id(id, session=None): + """ + Returns DBBinary object for given C{id} + + @type id: int + @param id: Id of the required binary -def binary_from_id(id): - return DBConn().session().query(Binary).filter_by(binary_id=id).one() + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: DBBinary + @return: DBBinary object for the given binary (None if not present) + """ + if session is None: + session = DBConn().session() + q = session.query(DBBinary).filter_by(binary_id=id) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_binary_from_id') + +def get_binaries_from_name(package, session=None): + """ + Returns list of DBBinary objects for given C{package} name + + @type package: str + @param package: DBBinary package name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: list of DBBinary objects for the given name (may be empty) + """ + if session is None: + session = DBConn().session() + return session.query(DBBinary).filter_by(package=package).all() + +__all__.append('get_binaries_from_name') + +################################################################################ class Component(object): def __init__(self, *args, **kwargs): @@ -99,6 +219,32 @@ class Component(object): def __repr__(self): return '' % self.component_name + +__all__.append('Component') + +def get_component(component, session=None): + """ + Returns database id for given C{component}. + + @type component: string + @param component: The name of the override type + + @rtype: int + @return: the database id for the given component + + """ + component = component.lower() + if session is None: + session = DBConn().session() + q = session.query(Component).filter_by(component_name=component) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_component') + +################################################################################ + class DBConfig(object): def __init__(self, *args, **kwargs): pass @@ -106,6 +252,10 @@ class DBConfig(object): def __repr__(self): return '' % self.name +__all__.append('DBConfig') + +################################################################################ + class ContentFilename(object): def __init__(self, *args, **kwargs): pass @@ -113,6 +263,44 @@ class ContentFilename(object): def __repr__(self): return '' % self.filename +__all__.append('ContentFilename') + +def get_or_set_contents_file_id(filename, session=None): + """ + Returns database id for given filename. + + If no matching file is found, a row is inserted. + + @type filename: string + @param filename: The filename + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: int + @return: the database id for the given component + """ + if session is None: + session = DBConn().session() + + try: + q = session.query(ContentFilename).filter_by(filename=filename) + if q.count() < 1: + cf = ContentFilename() + cf.filename = filename + session.add(cf) + return cf.cafilename_id + else: + return q.one().cafilename_id + + except: + traceback.print_exc() + raise + +__all__.append('get_or_set_contents_file_id') + +################################################################################ + class ContentFilepath(object): def __init__(self, *args, **kwargs): pass @@ -120,13 +308,105 @@ class ContentFilepath(object): def __repr__(self): return '' % self.filepath -class ContentAssociations(object): +__all__.append('ContentFilepath') + +def get_or_set_contents_path_id(filepath, session): + """ + Returns database id for given path. + + If no matching file is found, a row is inserted. + + @type filename: string + @param filename: The filepath + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: int + @return: the database id for the given path + """ + if session is None: + session = DBConn().session() + + try: + q = session.query(ContentFilepath).filter_by(filepath=filepath) + if q.count() < 1: + cf = ContentFilepath() + cf.filepath = filepath + session.add(cf) + return cf.cafilepath_id + else: + return q.one().cafilepath_id + + except: + traceback.print_exc() + raise + +__all__.append('get_or_set_contents_path_id') + +################################################################################ + +class ContentAssociation(object): def __init__(self, *args, **kwargs): pass def __repr__(self): return '' % self.ca_id +__all__.append('ContentAssociation') + +def insert_content_paths(binary_id, fullpaths, session=None): + """ + Make sure given path is associated with given binary id + + @type binary_id: int + @param binary_id: the id of the binary + @type fullpaths: list + @param fullpaths: the list of paths of the file being associated with the binary + @type session: SQLAlchemy session + @param session: Optional SQLAlchemy session. If this is passed, the caller + is responsible for ensuring a transaction has begun and committing the + results or rolling back based on the result code. If not passed, a commit + will be performed at the end of the function + + @return: True upon success + """ + + privatetrans = False + + if session is None: + session = DBConn().session() + privatetrans = True + + try: + for fullpath in fullpaths: + (path, file) = os.path.split(fullpath) + + # Get the necessary IDs ... + ca = ContentAssociation() + ca.binary_id = binary_id + ca.filename_id = get_or_set_contents_file_id(file) + ca.filepath_id = get_or_set_contents_path_id(path) + session.add(ca) + + # Only commit if we set up the session ourself + if privatetrans: + session.commit() + + return True + except: + traceback.print_exc() + + # Only rollback if we set up the session ourself + if privatetrans: + session.rollback() + + return False + +__all__.append('insert_content_paths') + +################################################################################ + class DSCFile(object): def __init__(self, *args, **kwargs): pass @@ -134,6 +414,10 @@ class DSCFile(object): def __repr__(self): return '' % self.dscfile_id +__all__.append('DSCFile') + +################################################################################ + class PoolFile(object): def __init__(self, *args, **kwargs): pass @@ -141,6 +425,37 @@ class PoolFile(object): def __repr__(self): return '' % self.filename +__all__.append('PoolFile') + +def get_poolfile_by_name(filename, location_id=None, session=None): + """ + Returns an array of PoolFile objects for the given filename and + (optionally) location_id + + @type filename: string + @param filename: the filename of the file to check against the DB + + @type location_id: int + @param location_id: the id of the location to look in (optional) + + @rtype: array + @return: array of PoolFile objects + """ + + if session is not None: + session = DBConn().session() + + q = session.query(PoolFile).filter_by(filename=filename) + + if location_id is not None: + q = q.join(Location).filter_by(location_id=location_id) + + return q.all() + +__all__.append('get_poolfile_by_name') + +################################################################################ + class Fingerprint(object): def __init__(self, *args, **kwargs): pass @@ -148,6 +463,10 @@ class Fingerprint(object): def __repr__(self): return '' % self.fingerprint +__all__.append('Fingerprint') + +################################################################################ + class Keyring(object): def __init__(self, *args, **kwargs): pass @@ -155,6 +474,10 @@ class Keyring(object): def __repr__(self): return '' % self.keyring_name +__all__.append('Keyring') + +################################################################################ + class Location(object): def __init__(self, *args, **kwargs): pass @@ -162,6 +485,46 @@ class Location(object): def __repr__(self): return '' % (self.path, self.location_id) +__all__.append('Location') + +def get_location(location, component=None, archive=None, session=None): + """ + Returns Location object for the given combination of location, component + and archive + + @type location: string + @param location: the path of the location, e.g. I{/srv/ftp.debian.org/ftp/pool/} + + @type component: string + @param component: the component name (if None, no restriction applied) + + @type archive: string + @param archive_id: the archive name (if None, no restriction applied) + + @rtype: Location / None + @return: Either a Location object or None if one can't be found + """ + + if session is None: + session = DBConn().session() + + q = session.query(Location).filter_by(path=location) + + if archive is not None: + q = q.join(Archive).filter_by(archive_name=archive) + + if component is not None: + q = q.join(Component).filter_by(component_name=component) + + if q.count() < 1: + return None + else: + return q.one() + +__all__.append('get_location') + +################################################################################ + class Maintainer(object): def __init__(self, *args, **kwargs): pass @@ -169,6 +532,10 @@ class Maintainer(object): def __repr__(self): return '''''' % (self.name, self.maintainer_id) +__all__.append('Maintainer') + +################################################################################ + class Override(object): def __init__(self, *args, **kwargs): pass @@ -176,6 +543,10 @@ class Override(object): def __repr__(self): return '' % (self.package, self.suite_id) +__all__.append('Override') + +################################################################################ + class OverrideType(object): def __init__(self, *args, **kwargs): pass @@ -183,6 +554,34 @@ class OverrideType(object): def __repr__(self): return '' % self.overridetype +__all__.append('OverrideType') + +def get_override_type(override_type, session=None): + """ + Returns OverrideType object for given C{override type}. + + @type override_type: string + @param override_type: The name of the override type + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: int + @return: the database id for the given override type + + """ + if session is None: + session = DBConn().session() + q = session.query(Priority).filter_by(priority=priority) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_override_type') + +################################################################################ + class PendingContentAssociation(object): def __init__(self, *args, **kwargs): pass @@ -190,6 +589,76 @@ class PendingContentAssociation(object): def __repr__(self): return '' % self.pca_id +__all__.append('PendingContentAssociation') + +def insert_pending_content_paths(package, fullpaths, session=None): + """ + Make sure given paths are temporarily associated with given + package + + @type package: dict + @param package: the package to associate with should have been read in from the binary control file + @type fullpaths: list + @param fullpaths: the list of paths of the file being associated with the binary + @type session: SQLAlchemy session + @param session: Optional SQLAlchemy session. If this is passed, the caller + is responsible for ensuring a transaction has begun and committing the + results or rolling back based on the result code. If not passed, a commit + will be performed at the end of the function + + @return: True upon success, False if there is a problem + """ + + privatetrans = False + + if session is None: + session = DBConn().session() + privatetrans = True + + try: + arch = get_architecture(package['Architecture'], session) + arch_id = arch.arch_id + + # Remove any already existing recorded files for this package + q = session.query(PendingContentAssociation) + q = q.filter_by(package=package['Package']) + q = q.filter_by(version=package['Version']) + q = q.filter_by(architecture=arch_id) + q.delete() + + # Insert paths + for fullpath in fullpaths: + (path, file) = os.path.split(fullpath) + + if path.startswith( "./" ): + path = path[2:] + + pca = PendingContentAssociation() + pca.package = package['Package'] + pca.version = package['Version'] + pca.filename_id = get_or_set_contents_file_id(file, session) + pca.filepath_id = get_or_set_contents_path_id(path, session) + pca.architecture = arch_id + session.add(pca) + + # Only commit if we set up the session ourself + if privatetrans: + session.commit() + + return True + except: + traceback.print_exc() + + # Only rollback if we set up the session ourself + if privatetrans: + session.rollback() + + return False + +__all__.append('insert_pending_content_paths') + +################################################################################ + class Priority(object): def __init__(self, *args, **kwargs): pass @@ -197,6 +666,34 @@ class Priority(object): def __repr__(self): return '' % (self.priority, self.priority_id) +__all__.append('Priority') + +def get_priority(priority, session=None): + """ + Returns Priority object for given C{priority name}. + + @type priority: string + @param priority: The name of the priority + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Priority + @return: Priority object for the given priority + + """ + if session is None: + session = DBConn().session() + q = session.query(Priority).filter_by(priority=priority) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_priority') + +################################################################################ + class Queue(object): def __init__(self, *args, **kwargs): pass @@ -204,6 +701,10 @@ class Queue(object): def __repr__(self): return '' % self.queue_name +__all__.append('Queue') + +################################################################################ + class QueueBuild(object): def __init__(self, *args, **kwargs): pass @@ -211,6 +712,10 @@ class QueueBuild(object): def __repr__(self): return '' % (self.filename, self.queue_id) +__all__.append('QueueBuild') + +################################################################################ + class Section(object): def __init__(self, *args, **kwargs): pass @@ -218,12 +723,93 @@ class Section(object): def __repr__(self): return '
' % self.section -class Source(object): +__all__.append('Section') + +def get_section(section, session=None): + """ + Returns Section object for given C{section name}. + + @type section: string + @param section: The name of the section + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Section + @return: Section object for the given section name + + """ + if session is None: + session = DBConn().session() + q = session.query(Section).filter_by(section=section) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_section') + +################################################################################ + +class DBSource(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % (self.source, self.version) + return '' % (self.source, self.version) + +__all__.append('DBSource') + +def get_sources_from_name(source, session=None): + """ + Returns list of DBSource objects for given C{source} name + + @type source: str + @param source: DBSource package name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: list of DBSource objects for the given name (may be empty) + """ + if session is None: + session = DBConn().session() + return session.query(DBSource).filter_by(source=source).all() + +__all__.append('get_sources_from_name') + +def get_source_in_suite(source, suite, session=None): + """ + Returns list of DBSource objects for a combination of C{source} and C{suite}. + + - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} + - B{suite} - a suite name, eg. I{unstable} + + @type source: string + @param source: source package name + + @type suite: string + @param suite: the suite name + + @rtype: string + @return: the version for I{source} in I{suite} + + """ + if session is None: + session = DBConn().session() + q = session.query(SrcAssociation) + q = q.join('source').filter_by(source=source) + q = q.join('suite').filter_by(suite_name=suite) + if q.count() == 0: + return None + # ???: Maybe we should just return the SrcAssociation object instead + return q.one().source + +__all__.append('get_source_in_suite') + +################################################################################ class SrcAssociation(object): def __init__(self, *args, **kwargs): @@ -232,6 +818,10 @@ class SrcAssociation(object): def __repr__(self): return '' % (self.sa_id, self.source, self.suite) +__all__.append('SrcAssociation') + +################################################################################ + class SrcUploader(object): def __init__(self, *args, **kwargs): pass @@ -239,6 +829,10 @@ class SrcUploader(object): def __repr__(self): return '' % self.uploader_id +__all__.append('SrcUploader') + +################################################################################ + class Suite(object): def __init__(self, *args, **kwargs): pass @@ -246,6 +840,65 @@ class Suite(object): def __repr__(self): return '' % self.suite_name +__all__.append('Suite') + +def get_suite_architecture(suite, architecture, session=None): + """ + Returns a SuiteArchitecture object given C{suite} and ${arch} or None if it + doesn't exist + + @type suite: str + @param suite: Suite name to search for + + @type architecture: str + @param architecture: Architecture name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: SuiteArchitecture + @return: the SuiteArchitecture object or None + """ + + if session is None: + session = DBConn().session() + + q = session.query(SuiteArchitecture) + q = q.join(Architecture).filter_by(arch_string=architecture) + q = q.join(Suite).filter_by(suite_name=suite) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_suite_architecture') + +def get_suite(suite, session=None): + """ + Returns Suite object for given C{suite name}. + + @type suite: string + @param suite: The name of the suite + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Suite + @return: Suite object for the requested suite name (None if not presenT) + + """ + if session is None: + session = DBConn().session() + q = session.query(Suite).filter_by(suite_name=suite) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_suite') + +################################################################################ + class SuiteArchitecture(object): def __init__(self, *args, **kwargs): pass @@ -253,6 +906,35 @@ class SuiteArchitecture(object): def __repr__(self): return '' % (self.suite_id, self.arch_id) +__all__.append('SuiteArchitecture') + +def get_suite_architectures(suite, session=None): + """ + Returns list of Architecture objects for given C{suite} name + + @type source: str + @param source: Suite name to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: list of Architecture objects for the given name (may be empty) + """ + + if session is None: + session = DBConn().session() + + q = session.query(Architecture) + q = q.join(SuiteArchitecture) + q = q.join(Suite).filter_by(suite_name=suite).order_by('arch_string') + return q.all() + +__all__.append('get_suite_architectures') + +################################################################################ + class Uid(object): def __init__(self, *args, **kwargs): pass @@ -260,6 +942,8 @@ class Uid(object): def __repr__(self): return '' % (self.uid, self.name) +__all__.append('Uid') + ################################################################################ class DBConn(Singleton): @@ -270,8 +954,10 @@ class DBConn(Singleton): super(DBConn, self).__init__(*args, **kwargs) def _startup(self, *args, **kwargs): + self.debug = False + if kwargs.has_key('debug'): + self.debug = True self.__createconn() - self.__init_caches() def __setuptables(self): self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) @@ -316,16 +1002,16 @@ class DBConn(Singleton): suite_id = self.tbl_bin_associations.c.suite, suite = relation(Suite), binary_id = self.tbl_bin_associations.c.bin, - binary = relation(Binary))) + binary = relation(DBBinary))) - mapper(Binary, self.tbl_binaries, + mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, package = self.tbl_binaries.c.package, version = self.tbl_binaries.c.version, maintainer_id = self.tbl_binaries.c.maintainer, maintainer = relation(Maintainer), source_id = self.tbl_binaries.c.source, - source = relation(Source), + source = relation(DBSource), arch_id = self.tbl_binaries.c.architecture, architecture = relation(Architecture), poolfile_id = self.tbl_binaries.c.file, @@ -344,14 +1030,14 @@ class DBConn(Singleton): mapper(DBConfig, self.tbl_config, properties = dict(config_id = self.tbl_config.c.id)) - mapper(ContentAssociations, self.tbl_content_associations, + mapper(ContentAssociation, self.tbl_content_associations, properties = dict(ca_id = self.tbl_content_associations.c.id, filename_id = self.tbl_content_associations.c.filename, filename = relation(ContentFilename), filepath_id = self.tbl_content_associations.c.filepath, filepath = relation(ContentFilepath), binary_id = self.tbl_content_associations.c.binary_pkg, - binary = relation(Binary))) + binary = relation(DBBinary))) mapper(ContentFilename, self.tbl_content_file_names, @@ -365,7 +1051,7 @@ class DBConn(Singleton): mapper(DSCFile, self.tbl_dsc_files, properties = dict(dscfile_id = self.tbl_dsc_files.c.id, source_id = self.tbl_dsc_files.c.source, - source = relation(Source), + source = relation(DBSource), poolfile_id = self.tbl_dsc_files.c.file, poolfile = relation(PoolFile))) @@ -434,7 +1120,7 @@ class DBConn(Singleton): mapper(Section, self.tbl_section, properties = dict(section_id = self.tbl_section.c.id)) - mapper(Source, self.tbl_source, + mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, version = self.tbl_source.c.version, maintainer_id = self.tbl_source.c.maintainer, @@ -457,12 +1143,12 @@ class DBConn(Singleton): suite_id = self.tbl_src_associations.c.suite, suite = relation(Suite), source_id = self.tbl_src_associations.c.source, - source = relation(Source))) + source = relation(DBSource))) mapper(SrcUploader, self.tbl_src_uploaders, properties = dict(uploader_id = self.tbl_src_uploaders.c.id, source_id = self.tbl_src_uploaders.c.source, - source = relation(Source, + source = relation(DBSource, primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)), maintainer_id = self.tbl_src_uploaders.c.maintainer, maintainer = relation(Maintainer, @@ -473,7 +1159,7 @@ class DBConn(Singleton): mapper(SuiteArchitecture, self.tbl_suite_architectures, properties = dict(suite_id = self.tbl_suite_architectures.c.suite, - suite = relation(Suite), + suite = relation(Suite, backref='suitearchitectures'), arch_id = self.tbl_suite_architectures.c.architecture, architecture = relation(Architecture))) @@ -496,12 +1182,12 @@ class DBConn(Singleton): if cnf["DB::Port"] and cnf["DB::Port"] != "-1": connstr += "?port=%s" % cnf["DB::Port"] - self.db_pg = create_engine(connstr) + self.db_pg = create_engine(connstr, echo=self.debug) self.db_meta = MetaData() self.db_meta.bind = self.db_pg self.db_smaker = sessionmaker(bind=self.db_pg, autoflush=True, - transactional=True) + autocommit=False) self.__setuptables() self.__setupmappers() @@ -509,517 +1195,5 @@ class DBConn(Singleton): def session(self): return self.db_smaker() - ## Cache functions - def __init_caches(self): - self.caches = {'suite': Cache(), - 'section': Cache(), - 'priority': Cache(), - 'override_type': Cache(), - 'architecture': Cache(), - 'archive': Cache(), - 'component': Cache(), - 'content_path_names': Cache(), - 'content_file_names': Cache(), - 'location': Cache(lambda x: '%s_%s_%s' % (x['location'], x['component'], x['location'])), - 'maintainer': {}, # TODO - 'keyring': {}, # TODO - 'source': Cache(lambda x: '%s_%s_' % (x['source'], x['version'])), - 'files': Cache(lambda x: '%s_%s_' % (x['filename'], x['location'])), - 'maintainer': {}, # TODO - 'fingerprint': {}, # TODO - 'queue': {}, # TODO - 'uid': {}, # TODO - 'suite_version': Cache(lambda x: '%s_%s' % (x['source'], x['suite'])), - } - - self.prepared_statements = {} - - def prepare(self,name,statement): - if not self.prepared_statements.has_key(name): - pgc.execute(statement) - self.prepared_statements[name] = statement - - def clear_caches(self): - self.__init_caches() - - ## Get functions - def __get_id(self, retfield, selectobj, cachekey, cachename=None): - # This is a bit of a hack but it's an internal function only - if cachename is not None: - res = self.caches[cachename].GetValue(cachekey) - if res: - return res - - c = selectobj.execute() - - if c.rowcount != 1: - return None - - res = c.fetchone() - - if retfield not in res.keys(): - return None - - res = res[retfield] - - if cachename is not None: - self.caches[cachename].SetValue(cachekey, res) - - return res - - def get_suite_id(self, suite): - """ - Returns database id for given C{suite}. - Results are kept in a cache during runtime to minimize database queries. - - @type suite: string - @param suite: The name of the suite - - @rtype: int - @return: the database id for the given suite - - """ - return int(self.__get_id('id', - self.tbl_suite.select(self.tbl_suite.columns.suite_name == suite), - suite, - 'suite')) - - def get_section_id(self, section): - """ - Returns database id for given C{section}. - Results are kept in a cache during runtime to minimize database queries. - - @type section: string - @param section: The name of the section - - @rtype: int - @return: the database id for the given section - - """ - return self.__get_id('id', - self.tbl_section.select(self.tbl_section.columns.section == section), - section, - 'section') - - def get_priority_id(self, priority): - """ - Returns database id for given C{priority}. - Results are kept in a cache during runtime to minimize database queries. - - @type priority: string - @param priority: The name of the priority - - @rtype: int - @return: the database id for the given priority - - """ - return self.__get_id('id', - self.tbl_priority.select(self.tbl_priority.columns.priority == priority), - priority, - 'priority') - - def get_override_type_id(self, override_type): - """ - Returns database id for given override C{type}. - Results are kept in a cache during runtime to minimize database queries. - - @type override_type: string - @param override_type: The name of the override type - - @rtype: int - @return: the database id for the given override type - - """ - return self.__get_id('id', - self.tbl_override_type.select(self.tbl_override_type.columns.type == override_type), - override_type, - 'override_type') - - def get_architecture_id(self, architecture): - """ - Returns database id for given C{architecture}. - Results are kept in a cache during runtime to minimize database queries. - - @type architecture: string - @param architecture: The name of the override type - - @rtype: int - @return: the database id for the given architecture - - """ - return self.__get_id('id', - self.tbl_architecture.select(self.tbl_architecture.columns.arch_string == architecture), - architecture, - 'architecture') - - def get_archive_id(self, archive): - """ - returns database id for given c{archive}. - results are kept in a cache during runtime to minimize database queries. - - @type archive: string - @param archive: the name of the override type - - @rtype: int - @return: the database id for the given archive - - """ - archive = archive.lower() - return self.__get_id('id', - self.tbl_archive.select(self.tbl_archive.columns.name == archive), - archive, - 'archive') - - def get_component_id(self, component): - """ - Returns database id for given C{component}. - Results are kept in a cache during runtime to minimize database queries. - - @type component: string - @param component: The name of the override type - - @rtype: int - @return: the database id for the given component - - """ - component = component.lower() - return self.__get_id('id', - self.tbl_component.select(self.tbl_component.columns.name == component), - component.lower(), - 'component') - - def get_location_id(self, location, component, archive): - """ - Returns database id for the location behind the given combination of - - B{location} - the path of the location, eg. I{/srv/ftp.debian.org/ftp/pool/} - - B{component} - the id of the component as returned by L{get_component_id} - - B{archive} - the id of the archive as returned by L{get_archive_id} - Results are kept in a cache during runtime to minimize database queries. - - @type location: string - @param location: the path of the location - - @type component: string - @param component: the name of the component - - @type archive: string - @param archive: the name of the archive - - @rtype: int - @return: the database id for the location - - """ - - archive = archive.lower() - component = component.lower() - - values = {'archive': archive, 'location': location, 'component': component} - - s = self.tbl_location.join(self.tbl_archive).join(self.tbl_component) - - s = s.select(self.tbl_location.columns.path == location) - s = s.where(self.tbl_archive.columns.name == archive) - s = s.where(self.tbl_component.columns.name == component) - - return self.__get_id('location.id', s, values, 'location') - - def get_source_id(self, source, version): - """ - Returns database id for the combination of C{source} and C{version} - - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} - - B{version} - Results are kept in a cache during runtime to minimize database queries. - - @type source: string - @param source: source package name - - @type version: string - @param version: the source version - - @rtype: int - @return: the database id for the source - - """ - s = self.tbl_source.select() - s = s.where(self.tbl_source.columns.source == source) - s = s.where(self.tbl_source.columns.version == version) - - return self.__get_id('id', s, {'source': source, 'version': version}, 'source') - - def get_suite(self, suite): - if isinstance(suite, str): - suite_id = self.get_suite_id(suite.lower()) - elif type(suite) == int: - suite_id = suite - - s = self.tbl_suite.select(self.tbl_suite.columns.id == suite_id) - c = s.execute() - if c.rowcount < 1: - return None - else: - return c.fetchone() - - def get_suite_version(self, source, suite): - """ - Returns database id for a combination of C{source} and C{suite}. - - - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} - - B{suite} - a suite name, eg. I{unstable} - - Results are kept in a cache during runtime to minimize database queries. - - @type source: string - @param source: source package name - - @type suite: string - @param suite: the suite name - - @rtype: string - @return: the version for I{source} in I{suite} - - """ - s = select([self.tbl_source.columns.source, self.tbl_source.columns.version]) -# s = self.tbl_source.join(self.tbl_src_associations).join(self.tbl_suite) - - s = s.select(self.tbl_suite.columns.suite_name == suite, use_labels=True) - s = s.select(self.tbl_source.columns.source == source) - - return self.__get_id('source.version', s, {'suite': suite, 'source': source}, 'suite_version') - - - def get_files_id (self, filename, size, md5sum, location_id): - """ - Returns -1, -2 or the file_id for filename, if its C{size} and C{md5sum} match an - existing copy. - - The database is queried using the C{filename} and C{location_id}. If a file does exist - at that location, the existing size and md5sum are checked against the provided - parameters. A size or checksum mismatch returns -2. If more than one entry is - found within the database, a -1 is returned, no result returns None, otherwise - the file id. - - Results are kept in a cache during runtime to minimize database queries. - - @type filename: string - @param filename: the filename of the file to check against the DB - - @type size: int - @param size: the size of the file to check against the DB - - @type md5sum: string - @param md5sum: the md5sum of the file to check against the DB - - @type location_id: int - @param location_id: the id of the location as returned by L{get_location_id} - - @rtype: int / None - @return: Various return values are possible: - - -2: size/checksum error - - -1: more than one file found in database - - None: no file found in database - - int: file id - - """ - values = {'filename' : filename, - 'location' : location_id} - - res = self.caches['files'].GetValue( values ) - - if not res: - query = """SELECT id, size, md5sum - FROM files - WHERE filename = %(filename)s AND location = %(location)s""" - - cursor = self.db_con.cursor() - cursor.execute( query, values ) - - if cursor.rowcount == 0: - res = None - - elif cursor.rowcount != 1: - res = -1 - - else: - row = cursor.fetchone() - - if row[1] != int(size) or row[2] != md5sum: - res = -2 - - else: - self.caches['files'].SetValue(values, row[0]) - res = row[0] - - return res - - - def get_or_set_contents_file_id(self, filename): - """ - Returns database id for given filename. - - Results are kept in a cache during runtime to minimize database queries. - If no matching file is found, a row is inserted. - - @type filename: string - @param filename: The filename - - @rtype: int - @return: the database id for the given component - """ - try: - values={'value': filename} - query = "SELECT id FROM content_file_names WHERE file = %(value)s" - id = self.__get_single_id(query, values, cachename='content_file_names') - if not id: - c = self.db_con.cursor() - c.execute( "INSERT INTO content_file_names VALUES (DEFAULT, %(value)s) RETURNING id", - values ) - - id = c.fetchone()[0] - self.caches['content_file_names'].SetValue(values, id) - - return id - except: - traceback.print_exc() - raise - - def get_or_set_contents_path_id(self, path): - """ - Returns database id for given path. - - Results are kept in a cache during runtime to minimize database queries. - If no matching file is found, a row is inserted. - - @type path: string - @param path: The filename - - @rtype: int - @return: the database id for the given component - """ - try: - values={'value': path} - query = "SELECT id FROM content_file_paths WHERE path = %(value)s" - id = self.__get_single_id(query, values, cachename='content_path_names') - if not id: - c = self.db_con.cursor() - c.execute( "INSERT INTO content_file_paths VALUES (DEFAULT, %(value)s) RETURNING id", - values ) - - id = c.fetchone()[0] - self.caches['content_path_names'].SetValue(values, id) - - return id - except: - traceback.print_exc() - raise - - def get_suite_architectures(self, suite): - """ - Returns list of architectures for C{suite}. - - @type suite: string, int - @param suite: the suite name or the suite_id - - @rtype: list - @return: the list of architectures for I{suite} - """ - - suite_id = None - if type(suite) == str: - suite_id = self.get_suite_id(suite) - elif type(suite) == int: - suite_id = suite - else: - return None - - c = self.db_con.cursor() - c.execute( """SELECT a.arch_string FROM suite_architectures sa - JOIN architecture a ON (a.id = sa.architecture) - WHERE suite='%s'""" % suite_id ) - - return map(lambda x: x[0], c.fetchall()) - - def insert_content_paths(self, bin_id, fullpaths): - """ - Make sure given path is associated with given binary id - - @type bin_id: int - @param bin_id: the id of the binary - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - - @return: True upon success - """ - - c = self.db_con.cursor() - - c.execute("BEGIN WORK") - try: - - for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) - - if path.startswith( "./" ): - path = path[2:] - # Get the necessary IDs ... - file_id = self.get_or_set_contents_file_id(file) - path_id = self.get_or_set_contents_path_id(path) - - c.execute("""INSERT INTO content_associations - (binary_pkg, filepath, filename) - VALUES ( '%d', '%d', '%d')""" % (bin_id, path_id, file_id) ) - - c.execute("COMMIT") - return True - except: - traceback.print_exc() - c.execute("ROLLBACK") - return False - - def insert_pending_content_paths(self, package, fullpaths): - """ - Make sure given paths are temporarily associated with given - package - - @type package: dict - @param package: the package to associate with should have been read in from the binary control file - @type fullpaths: list - @param fullpaths: the list of paths of the file being associated with the binary - - @return: True upon success - """ - - c = self.db_con.cursor() - - c.execute("BEGIN WORK") - try: - arch_id = self.get_architecture_id(package['Architecture']) - - # Remove any already existing recorded files for this package - c.execute("""DELETE FROM pending_content_associations - WHERE package=%(Package)s - AND version=%(Version)s - AND architecture=%(ArchID)s""", {'Package': package['Package'], - 'Version': package['Version'], - 'ArchID': arch_id}) - - for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) - - if path.startswith( "./" ): - path = path[2:] - # Get the necessary IDs ... - file_id = self.get_or_set_contents_file_id(file) - path_id = self.get_or_set_contents_path_id(path) - - c.execute("""INSERT INTO pending_content_associations - (package, version, architecture, filepath, filename) - VALUES (%%(Package)s, %%(Version)s, '%d', '%d', '%d')""" - % (arch_id, path_id, file_id), package ) - - c.execute("COMMIT") - return True - except: - traceback.print_exc() - c.execute("ROLLBACK") - return False +__all__.append('DBConn')