X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=f32a94bb91d550fa82886943a80c83d620ec613e;hb=4db4a1577f7971458d41966083b82e46d50b8a04;hp=32fbdb5526376e9dcdd0ca2af74756871fd1fa38;hpb=d612affcdd3237f6906189dc0f27edc09850f4ed;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 32fbdb55..f32a94bb 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -43,11 +43,15 @@ from sqlalchemy.orm import sessionmaker, mapper, relation # Don't remove this, we re-export the exceptions to scripts which import us from sqlalchemy.exc import * +# Only import Config until Queue stuff is changed to store its config +# in the database +from config import Config from singleton import Singleton +from textutils import fix_maintainer ################################################################################ -__all__ = [] +__all__ = ['IntegrityError', 'SQLAlchemyError'] ################################################################################ @@ -55,6 +59,18 @@ class Architecture(object): def __init__(self, *args, **kwargs): pass + def __eq__(self, val): + if isinstance(val, str): + return (self.arch_string== val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.arch_string != val) + # This signals to use the normal comparison operator + return NotImplemented + def __repr__(self): return '' % self.arch_string @@ -158,18 +174,36 @@ __all__.append('BinAssociation') ################################################################################ -class Binary(object): +class DBBinary(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % (self.package, self.version, self.architecture) + return '' % (self.package, self.version, self.architecture) + +__all__.append('DBBinary') + +def get_suites_binary_in(package, session=None): + """ + Returns list of Suite objects which given C{package} name is in + + @type source: str + @param source: DBBinary package name to search for + + @rtype: list + @return: list of Suite objects for the given package + """ + + if session is None: + session = DBConn().session() + + return session.query(Suite).join(BinAssociation).join(DBBinary).filter_by(package=package).all() -__all__.append('Binary') +__all__.append('get_suites_binary_in') def get_binary_from_id(id, session=None): """ - Returns Binary object for given C{id} + Returns DBBinary object for given C{id} @type id: int @param id: Id of the required binary @@ -178,44 +212,133 @@ def get_binary_from_id(id, session=None): @param session: Optional SQLA session object (a temporary one will be generated if not supplied) - @rtype: Binary - @return: Binary object for the given binary (None if not present) + @rtype: DBBinary + @return: DBBinary object for the given binary (None if not present) """ if session is None: session = DBConn().session() - q = session.query(Binary).filter_by(binary_id=id) + q = session.query(DBBinary).filter_by(binary_id=id) if q.count() == 0: return None return q.one() __all__.append('get_binary_from_id') -def get_binaries_from_name(package, session=None): +def get_binaries_from_name(package, version=None, architecture=None, session=None): """ - Returns list of Binary objects for given C{package} name + Returns list of DBBinary objects for given C{package} name @type package: str - @param package: Binary package name to search for + @param package: DBBinary package name to search for + + @type version: str or None + @param version: Version to search for (or None) + + @type package: str, list or None + @param package: Architectures to limit to (or None if no limit) @type session: Session @param session: Optional SQL session object (a temporary one will be generated if not supplied) @rtype: list - @return: list of Binary objects for the given name (may be empty) + @return: list of DBBinary objects for the given name (may be empty) """ if session is None: session = DBConn().session() - return session.query(Binary).filter_by(package=package).all() + + q = session.query(DBBinary).filter_by(package=package) + + if version is not None: + q = q.filter_by(version=version) + + if architecture is not None: + if not isinstance(architecture, list): + architecture = [architecture] + q = q.join(Architecture).filter(Architecture.arch_string.in_(architecture)) + + return q.all() __all__.append('get_binaries_from_name') +def get_binaries_from_source_id(source_id, session=None): + """ + Returns list of DBBinary objects for given C{source_id} + + @type source_id: int + @param source_id: source_id to search for + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: list of DBBinary objects for the given name (may be empty) + """ + if session is None: + session = DBConn().session() + return session.query(DBBinary).filter_by(source_id=source_id).all() + +__all__.append('get_binaries_from_source_id') + + +def get_binary_from_name_suite(package, suitename, session=None): + ### For dak examine-package + ### XXX: Doesn't use object API yet + if session is None: + session = DBConn().session() + + sql = """SELECT DISTINCT(b.package), b.version, c.name, su.suite_name + FROM binaries b, files fi, location l, component c, bin_associations ba, suite su + WHERE b.package=:package + AND b.file = fi.id + AND fi.location = l.id + AND l.component = c.id + AND ba.bin=b.id + AND ba.suite = su.id + AND su.suite_name=:suitename + ORDER BY b.version DESC""" + + return session.execute(sql, {'package': package, 'suitename': suitename}) + +__all__.append('get_binary_from_name_suite') + +def get_binary_components(package, suitename, arch, session=None): +# Check for packages that have moved from one component to another + query = """SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f + WHERE b.package=:package AND s.suite_name=:suitename + AND (a.arch_string = :arch OR a.arch_string = 'all') + AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id + AND f.location = l.id + AND l.component = c.id + AND b.file = f.id""" + + vals = {'package': package, 'suitename': suitename, 'arch': arch} + + if session is None: + session = DBConn().session() + return session.execute(query, vals) + +__all__.append('get_binary_components') + ################################################################################ class Component(object): def __init__(self, *args, **kwargs): pass + def __eq__(self, val): + if isinstance(val, str): + return (self.component_name == val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.component_name != val) + # This signals to use the normal comparison operator + return NotImplemented + def __repr__(self): return '' % self.component_name @@ -275,13 +398,16 @@ def get_or_set_contents_file_id(filename, session=None): @param filename: The filename @type session: SQLAlchemy @param session: Optional SQL session object (a temporary one will be - generated if not supplied) + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. @rtype: int @return: the database id for the given component """ + privatetrans = False if session is None: session = DBConn().session() + privatetrans = True try: q = session.query(ContentFilename).filter_by(filename=filename) @@ -289,6 +415,11 @@ def get_or_set_contents_file_id(filename, session=None): cf = ContentFilename() cf.filename = filename session.add(cf) + if privatetrans: + session.commit() + session.close() + else: + session.flush() return cf.cafilename_id else: return q.one().cafilename_id @@ -299,6 +430,59 @@ def get_or_set_contents_file_id(filename, session=None): __all__.append('get_or_set_contents_file_id') +def get_contents(suite, overridetype, section=None, session=None): + """ + Returns contents for a suite / overridetype combination, limiting + to a section if not None. + + @type suite: Suite + @param suite: Suite object + + @type overridetype: OverrideType + @param overridetype: OverrideType object + + @type section: Section + @param section: Optional section object to limit results to + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: ResultsProxy + @return: ResultsProxy object set up to return tuples of (filename, section, + package, arch_id) + """ + + if session is None: + session = DBConn().session() + + # find me all of the contents for a given suite + contents_q = """SELECT (p.path||'/'||n.file) AS fn, + s.section, + b.package, + b.architecture + FROM content_associations c join content_file_paths p ON (c.filepath=p.id) + JOIN content_file_names n ON (c.filename=n.id) + JOIN binaries b ON (b.id=c.binary_pkg) + JOIN override o ON (o.package=b.package) + JOIN section s ON (s.id=o.section) + WHERE o.suite = :suiteid AND o.type = :overridetypeid + AND b.type=:overridetypename""" + + vals = {'suiteid': suite.suite_id, + 'overridetypeid': overridetype.overridetype_id, + 'overridetypename': overridetype.overridetype} + + if section is not None: + contents_q += " AND s.id = :sectionid" + vals['sectionid'] = section.section_id + + contents_q += " ORDER BY fn" + + return session.execute(contents_q, vals) + +__all__.append('get_contents') + ################################################################################ class ContentFilepath(object): @@ -310,7 +494,7 @@ class ContentFilepath(object): __all__.append('ContentFilepath') -def get_or_set_contents_path_id(filepath, session): +def get_or_set_contents_path_id(filepath, session=None): """ Returns database id for given path. @@ -320,13 +504,16 @@ def get_or_set_contents_path_id(filepath, session): @param filename: The filepath @type session: SQLAlchemy @param session: Optional SQL session object (a temporary one will be - generated if not supplied) + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. @rtype: int @return: the database id for the given path """ + privatetrans = False if session is None: session = DBConn().session() + privatetrans = True try: q = session.query(ContentFilepath).filter_by(filepath=filepath) @@ -334,6 +521,11 @@ def get_or_set_contents_path_id(filepath, session): cf = ContentFilepath() cf.filepath = filepath session.add(cf) + if privatetrans: + session.commit() + session.close() + else: + session.flush() return cf.cafilepath_id else: return q.one().cafilepath_id @@ -367,7 +559,8 @@ def insert_content_paths(binary_id, fullpaths, session=None): @param session: Optional SQLAlchemy session. If this is passed, the caller is responsible for ensuring a transaction has begun and committing the results or rolling back based on the result code. If not passed, a commit - will be performed at the end of the function + will be performed at the end of the function, otherwise the caller is + responsible for commiting. @return: True upon success """ @@ -379,27 +572,40 @@ def insert_content_paths(binary_id, fullpaths, session=None): privatetrans = True try: + # Insert paths + pathcache = {} for fullpath in fullpaths: + # Get the necessary IDs ... (path, file) = os.path.split(fullpath) - # Get the necessary IDs ... + filepath_id = get_or_set_contents_path_id(path, session) + filename_id = get_or_set_contents_file_id(file, session) + + pathcache[fullpath] = (filepath_id, filename_id) + + for fullpath, dat in pathcache.items(): ca = ContentAssociation() ca.binary_id = binary_id - ca.filename_id = get_or_set_contents_file_id(file) - ca.filepath_id = get_or_set_contents_path_id(path) + ca.filepath_id = dat[0] + ca.filename_id = dat[1] session.add(ca) # Only commit if we set up the session ourself if privatetrans: session.commit() + session.close() + else: + session.flush() return True + except: traceback.print_exc() # Only rollback if we set up the session ourself if privatetrans: session.rollback() + session.close() return False @@ -416,6 +622,41 @@ class DSCFile(object): __all__.append('DSCFile') +def get_dscfiles(dscfile_id=None, source_id=None, poolfile_id=None, session=None): + """ + Returns a list of DSCFiles which may be empty + + @type dscfile_id: int (optional) + @param dscfile_id: the dscfile_id of the DSCFiles to find + + @type source_id: int (optional) + @param source_id: the source id related to the DSCFiles to find + + @type poolfile_id: int (optional) + @param poolfile_id: the poolfile id related to the DSCFiles to find + + @rtype: list + @return: Possibly empty list of DSCFiles + """ + + if session is None: + session = DBConn().session() + + q = session.query(DSCFile) + + if dscfile_id is not None: + q = q.filter_by(dscfile_id=dscfile_id) + + if source_id is not None: + q = q.filter_by(source_id=source_id) + + if poolfile_id is not None: + q = q.filter_by(poolfile_id=poolfile_id) + + return q.all() + +__all__.append('get_dscfiles') + ################################################################################ class PoolFile(object): @@ -427,6 +668,76 @@ class PoolFile(object): __all__.append('PoolFile') +def check_poolfile(filename, filesize, md5sum, location_id, session=None): + """ + Returns a tuple: + (ValidFileFound [boolean or None], PoolFile object or None) + + @type filename: string + @param filename: the filename of the file to check against the DB + + @type filesize: int + @param filesize: the size of the file to check against the DB + + @type md5sum: string + @param md5sum: the md5sum of the file to check against the DB + + @type location_id: int + @param location_id: the id of the location to look in + + @rtype: tuple + @return: Tuple of length 2. + If more than one file found with that name: + (None, None) + If valid pool file found: (True, PoolFile object) + If valid pool file not found: + (False, None) if no file found + (False, PoolFile object) if file found with size/md5sum mismatch + """ + + if session is None: + session = DBConn().session() + + q = session.query(PoolFile).filter_by(filename=filename) + q = q.join(Location).filter_by(location_id=location_id) + + if q.count() > 1: + return (None, None) + if q.count() < 1: + return (False, None) + + obj = q.one() + if obj.md5sum != md5sum or obj.filesize != filesize: + return (False, obj) + + return (True, obj) + +__all__.append('check_poolfile') + +def get_poolfile_by_id(file_id, session=None): + """ + Returns a PoolFile objects or None for the given id + + @type file_id: int + @param file_id: the id of the file to look for + + @rtype: PoolFile or None + @return: either the PoolFile object or None + """ + + if session is None: + session = DBConn().session() + + q = session.query(PoolFile).filter_by(file_id=file_id) + + if q.count() > 0: + return q.one() + + return None + +__all__.append('get_poolfile_by_id') + + def get_poolfile_by_name(filename, location_id=None, session=None): """ Returns an array of PoolFile objects for the given filename and @@ -442,7 +753,7 @@ def get_poolfile_by_name(filename, location_id=None, session=None): @return: array of PoolFile objects """ - if session is not None: + if session is None: session = DBConn().session() q = session.query(PoolFile).filter_by(filename=filename) @@ -454,6 +765,27 @@ def get_poolfile_by_name(filename, location_id=None, session=None): __all__.append('get_poolfile_by_name') +def get_poolfile_like_name(filename, session=None): + """ + Returns an array of PoolFile objects which are like the given name + + @type filename: string + @param filename: the filename of the file to check against the DB + + @rtype: array + @return: array of PoolFile objects + """ + + if session is None: + session = DBConn().session() + + # TODO: There must be a way of properly using bind parameters with %FOO% + q = session.query(PoolFile).filter(PoolFile.filename.like('%%%s%%' % filename)) + + return q.all() + +__all__.append('get_poolfile_like_name') + ################################################################################ class Fingerprint(object): @@ -465,6 +797,49 @@ class Fingerprint(object): __all__.append('Fingerprint') +def get_or_set_fingerprint(fpr, session=None): + """ + Returns Fingerprint object for given fpr. + + If no matching fpr is found, a row is inserted. + + @type fpr: string + @param fpr: The fpr to find / add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. + A flush will be performed either way. + + @rtype: Fingerprint + @return: the Fingerprint object for the given fpr + """ + privatetrans = False + if session is None: + session = DBConn().session() + privatetrans = True + + try: + q = session.query(Fingerprint).filter_by(fingerprint=fpr) + if q.count() < 1: + fingerprint = Fingerprint() + fingerprint.fingerprint = fpr + session.add(fingerprint) + if privatetrans: + session.commit() + else: + session.flush() + return fingerprint + else: + return q.one() + + except: + traceback.print_exc() + raise + +__all__.append('get_or_set_fingerprint') + ################################################################################ class Keyring(object): @@ -532,8 +907,131 @@ class Maintainer(object): def __repr__(self): return '''''' % (self.name, self.maintainer_id) + def get_split_maintainer(self): + if not hasattr(self, 'name') or self.name is None: + return ('', '', '', '') + + return fix_maintainer(self.name.strip()) + __all__.append('Maintainer') +def get_or_set_maintainer(name, session=None): + """ + Returns Maintainer object for given maintainer name. + + If no matching maintainer name is found, a row is inserted. + + @type name: string + @param name: The maintainer name to add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. + A flush will be performed either way. + + @rtype: Maintainer + @return: the Maintainer object for the given maintainer + """ + privatetrans = False + if session is None: + session = DBConn().session() + privatetrans = True + + try: + q = session.query(Maintainer).filter_by(name=name) + if q.count() < 1: + maintainer = Maintainer() + maintainer.name = name + session.add(maintainer) + if privatetrans: + session.commit() + else: + session.flush() + return maintainer + else: + return q.one() + + except: + traceback.print_exc() + raise + +__all__.append('get_or_set_maintainer') + +################################################################################ + +class NewComment(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '''''' % (self.package, self.version, self.comment_id) + +__all__.append('NewComment') + +def has_new_comment(package, version, session=None): + """ + Returns true if the given combination of C{package}, C{version} has a comment. + + @type package: string + @param package: name of the package + + @type version: string + @param version: package version + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: boolean + @return: true/false + """ + + if session is None: + session = DBConn().session() + + q = session.query(NewComment) + q = q.filter_by(package=package) + q = q.filter_by(version=version) + return q.count() > 0 + +__all__.append('has_new_comment') + +def get_new_comments(package=None, version=None, comment_id=None, session=None): + """ + Returns (possibly empty) list of NewComment objects for the given + parameters + + @type package: string (optional) + @param package: name of the package + + @type version: string (optional) + @param version: package version + + @type comment_id: int (optional) + @param comment_id: An id of a comment + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: A (possibly empty) list of NewComment objects will be returned + + """ + + if session is None: + session = DBConn().session() + + q = session.query(NewComment) + if package is not None: q = q.filter_by(package=package) + if version is not None: q = q.filter_by(version=version) + if comment_id is not None: q = q.filter_by(comment_id=comment_id) + + return q.all() + +__all__.append('get_new_comments') + ################################################################################ class Override(object): @@ -545,6 +1043,56 @@ class Override(object): __all__.append('Override') +def get_override(package, suite=None, component=None, overridetype=None, session=None): + """ + Returns Override object for the given parameters + + @type package: string + @param package: The name of the package + + @type suite: string, list or None + @param suite: The name of the suite (or suites if a list) to limit to. If + None, don't limit. Defaults to None. + + @type component: string, list or None + @param component: The name of the component (or components if a list) to + limit to. If None, don't limit. Defaults to None. + + @type overridetype: string, list or None + @param overridetype: The name of the overridetype (or overridetypes if a list) to + limit to. If None, don't limit. Defaults to None. + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: list + @return: A (possibly empty) list of Override objects will be returned + + """ + if session is None: + session = DBConn().session() + + q = session.query(Override) + q = q.filter_by(package=package) + + if suite is not None: + if not isinstance(suite, list): suite = [suite] + q = q.join(Suite).filter(Suite.suite_name.in_(suite)) + + if component is not None: + if not isinstance(component, list): component = [component] + q = q.join(Component).filter(Component.component_name.in_(component)) + + if overridetype is not None: + if not isinstance(overridetype, list): overridetype = [overridetype] + q = q.join(OverrideType).filter(OverrideType.overridetype.in_(overridetype)) + + return q.all() + +__all__.append('get_override') + + ################################################################################ class OverrideType(object): @@ -573,7 +1121,7 @@ def get_override_type(override_type, session=None): """ if session is None: session = DBConn().session() - q = session.query(Priority).filter_by(priority=priority) + q = session.query(OverrideType).filter_by(overridetype=override_type) if q.count() == 0: return None return q.one() @@ -627,23 +1175,32 @@ def insert_pending_content_paths(package, fullpaths, session=None): q.delete() # Insert paths + pathcache = {} for fullpath in fullpaths: (path, file) = os.path.split(fullpath) if path.startswith( "./" ): path = path[2:] + filepath_id = get_or_set_contents_path_id(path, session) + filename_id = get_or_set_contents_file_id(file, session) + + pathcache[fullpath] = (filepath_id, filename_id) + + for fullpath, dat in pathcache.items(): pca = PendingContentAssociation() pca.package = package['Package'] pca.version = package['Version'] - pca.filename_id = get_or_set_contents_file_id(file, session) - pca.filepath_id = get_or_set_contents_path_id(path, session) + pca.filepath_id = dat[0] + pca.filename_id = dat[1] pca.architecture = arch_id session.add(pca) # Only commit if we set up the session ourself if privatetrans: session.commit() + else: + session.flush() return True except: @@ -663,6 +1220,18 @@ class Priority(object): def __init__(self, *args, **kwargs): pass + def __eq__(self, val): + if isinstance(val, str): + return (self.priority == val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.priority != val) + # This signals to use the normal comparison operator + return NotImplemented + def __repr__(self): return '' % (self.priority, self.priority_id) @@ -692,6 +1261,29 @@ def get_priority(priority, session=None): __all__.append('get_priority') +def get_priorities(session=None): + """ + Returns dictionary of priority names -> id mappings + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: dictionary + @return: dictionary of priority names -> id mappings + """ + if session is None: + session = DBConn().session() + + ret = {} + q = session.query(Priority) + for x in q.all(): + ret[x.priority] = x.priority_id + + return ret + +__all__.append('get_priorities') + ################################################################################ class Queue(object): @@ -701,8 +1293,143 @@ class Queue(object): def __repr__(self): return '' % self.queue_name + def autobuild_upload(self, changes, srcpath, session=None): + """ + Update queue_build database table used for incoming autobuild support. + + @type changes: Changes + @param changes: changes object for the upload to process + + @type srcpath: string + @param srcpath: path for the queue file entries/link destinations + + @type session: SQLAlchemy session + @param session: Optional SQLAlchemy session. If this is passed, the + caller is responsible for ensuring a transaction has begun and + committing the results or rolling back based on the result code. If + not passed, a commit will be performed at the end of the function, + otherwise the caller is responsible for commiting. + + @rtype: NoneType or string + @return: None if the operation failed, a string describing the error if not + """ + + localcommit = False + if session is None: + session = DBConn().session() + localcommit = True + + # TODO: Remove by moving queue config into the database + conf = Config() + + for suitename in changes.changes["distribution"].keys(): + # TODO: Move into database as: + # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build) + # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink) + # This also gets rid of the SecurityQueueBuild hack below + if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"): + continue + + # Find suite object + s = get_suite(suitename, session) + if s is None: + return "INTERNAL ERROR: Could not find suite %s" % suitename + + # TODO: Get from database as above + dest_dir = conf["Dir::QueueBuild"] + + # TODO: Move into database as above + if conf.FindB("Dinstall::SecurityQueueBuild"): + dest_dir = os.path.join(dest_dir, suitename) + + for file_entry in changes.files.keys(): + src = os.path.join(srcpath, file_entry) + dest = os.path.join(dest_dir, file_entry) + + # TODO: Move into database as above + if conf.FindB("Dinstall::SecurityQueueBuild"): + # Copy it since the original won't be readable by www-data + utils.copy(src, dest) + else: + # Create a symlink to it + os.symlink(src, dest) + + qb = QueueBuild() + qb.suite_id = s.suite_id + qb.queue_id = self.queue_id + qb.filename = dest + qb.in_queue = True + + session.add(qb) + + # If the .orig.tar.gz is in the pool, create a symlink to + # it (if one doesn't already exist) + if changes.orig_tar_id: + # Determine the .orig.tar.gz file name + for dsc_file in changes.dsc_files.keys(): + if dsc_file.endswith(".orig.tar.gz"): + filename = dsc_file + + dest = os.path.join(dest_dir, filename) + + # If it doesn't exist, create a symlink + if not os.path.exists(dest): + q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", + {'id': changes.orig_tar_id}) + res = q.fetchone() + if not res: + return "[INTERNAL ERROR] Couldn't find id %s in files table." % (changes.orig_tar_id) + + src = os.path.join(res[0], res[1]) + os.symlink(src, dest) + + # Add it to the list of packages for later processing by apt-ftparchive + qb = QueueBuild() + qb.suite_id = s.suite_id + qb.queue_id = self.queue_id + qb.filename = dest + qb.in_queue = True + session.add(qb) + + # If it does, update things to ensure it's not removed prematurely + else: + qb = get_queue_build(dest, suite_id, session) + if qb is None: + qb.in_queue = True + qb.last_used = None + session.add(qb) + + if localcommit: + session.commit() + + return None + __all__.append('Queue') +def get_queue(queuename, session=None): + """ + Returns Queue object for given C{queue name}. + + @type queuename: string + @param queuename: The name of the queue + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Queue + @return: Queue object for the given queue + + """ + if session is None: + session = DBConn().session() + q = session.query(Queue).filter_by(queue_name=queuename) + if q.count() == 0: + return None + return q.one() + +__all__.append('get_queue') + ################################################################################ class QueueBuild(object): @@ -714,12 +1441,56 @@ class QueueBuild(object): __all__.append('QueueBuild') +def get_queue_build(filename, suite, session=None): + """ + Returns QueueBuild object for given C{filename} and C{suite}. + + @type filename: string + @param filename: The name of the file + + @type suiteid: int or str + @param suiteid: Suite name or ID + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Queue + @return: Queue object for the given queue + + """ + if session is None: + session = DBConn().session() + if isinstance(suite, int): + q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite) + else: + q = session.query(QueueBuild).filter_by(filename=filename) + q = q.join(Suite).filter_by(suite_name=suite) + + if q.count() == 0: + return None + return q.one() + +__all__.append('get_queue_build') + ################################################################################ class Section(object): def __init__(self, *args, **kwargs): pass + def __eq__(self, val): + if isinstance(val, str): + return (self.section == val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.section != val) + # This signals to use the normal comparison operator + return NotImplemented + def __repr__(self): return '
' % self.section @@ -749,40 +1520,167 @@ def get_section(section, session=None): __all__.append('get_section') +def get_sections(session=None): + """ + Returns dictionary of section names -> id mappings + + @type session: Session + @param session: Optional SQL session object (a temporary one will be + generated if not supplied) + + @rtype: dictionary + @return: dictionary of section names -> id mappings + """ + if session is None: + session = DBConn().session() + + ret = {} + q = session.query(Section) + for x in q.all(): + ret[x.section] = x.section_id + + return ret + +__all__.append('get_sections') + ################################################################################ -class Source(object): +class DBSource(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % (self.source, self.version) + return '' % (self.source, self.version) + +__all__.append('DBSource') + +def source_exists(source, source_version, suites = ["any"], session=None): + """ + Ensure that source exists somewhere in the archive for the binary + upload being processed. + 1. exact match => 1.0-3 + 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 + + @type package: string + @param package: package source name + + @type source_version: string + @param source_version: expected source version + + @type suites: list + @param suites: list of suites to check in, default I{any} -__all__.append('Source') + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: int + @return: returns 1 if a source with expected version is found, otherwise 0 + + """ -def get_sources_from_name(source, session=None): + if session is None: + session = DBConn().session() + + cnf = Config() + + for suite in suites: + q = session.query(DBSource).filter_by(source=source) + if suite != "any": + # source must exist in suite X, or in some other suite that's + # mapped to X, recursively... silent-maps are counted too, + # unreleased-maps aren't. + maps = cnf.ValueList("SuiteMappings")[:] + maps.reverse() + maps = [ m.split() for m in maps ] + maps = [ (x[1], x[2]) for x in maps + if x[0] == "map" or x[0] == "silent-map" ] + s = [suite] + for x in maps: + if x[1] in s and x[0] not in s: + s.append(x[0]) + + q = q.join(SrcAssociation).join(Suite) + q = q.filter(Suite.suite_name.in_(s)) + + # Reduce the query results to a list of version numbers + ql = [ j.version for j in q.all() ] + + # Try (1) + if source_version in ql: + continue + + # Try (2) + from daklib.regexes import re_bin_only_nmu + orig_source_version = re_bin_only_nmu.sub('', source_version) + if orig_source_version in ql: + continue + + # No source found so return not ok + return 0 + + # We're good + return 1 + +__all__.append('source_exists') + +def get_suites_source_in(source, session=None): """ - Returns list of Source objects for given C{source} name + Returns list of Suite objects which given C{source} name is in @type source: str - @param source: Source package name to search for + @param source: DBSource package name to search for + + @rtype: list + @return: list of Suite objects for the given source + """ + + if session is None: + session = DBConn().session() + + return session.query(Suite).join(SrcAssociation).join(DBSource).filter_by(source=source).all() + +__all__.append('get_suites_source_in') + +def get_sources_from_name(source, version=None, dm_upload_allowed=None, session=None): + """ + Returns list of DBSource objects for given C{source} name and other parameters + + @type source: str + @param source: DBSource package name to search for + + @type source: str or None + @param source: DBSource version name to search for or None if not applicable + + @type dm_upload_allowed: bool + @param dm_upload_allowed: If None, no effect. If True or False, only + return packages with that dm_upload_allowed setting @type session: Session @param session: Optional SQL session object (a temporary one will be generated if not supplied) @rtype: list - @return: list of Source objects for the given name (may be empty) + @return: list of DBSource objects for the given name (may be empty) """ if session is None: session = DBConn().session() - return session.query(Source).filter_by(source=source).all() + + q = session.query(DBSource).filter_by(source=source) + + if version is not None: + q = q.filter_by(version=version) + + if dm_upload_allowed is not None: + q = q.filter_by(dm_upload_allowed=dm_upload_allowed) + + return q.all() __all__.append('get_sources_from_name') def get_source_in_suite(source, suite, session=None): """ - Returns list of Source objects for a combination of C{source} and C{suite}. + Returns list of DBSource objects for a combination of C{source} and C{suite}. - B{source} - source package name, eg. I{mailfilter}, I{bbdb}, I{glibc} - B{suite} - a suite name, eg. I{unstable} @@ -833,6 +1731,26 @@ __all__.append('SrcUploader') ################################################################################ +SUITE_FIELDS = [ ('SuiteName', 'suite_name'), + ('SuiteID', 'suite_id'), + ('Version', 'version'), + ('Origin', 'origin'), + ('Label', 'label'), + ('Description', 'description'), + ('Untouchable', 'untouchable'), + ('Announce', 'announce'), + ('Codename', 'codename'), + ('OverrideCodename', 'overridecodename'), + ('ValidTime', 'validtime'), + ('Priority', 'priority'), + ('NotAutomatic', 'notautomatic'), + ('CopyChanges', 'copychanges'), + ('CopyDotDak', 'copydotdak'), + ('CommentsDir', 'commentsdir'), + ('OverrideSuite', 'overridesuite'), + ('ChangelogBase', 'changelogbase')] + + class Suite(object): def __init__(self, *args, **kwargs): pass @@ -840,6 +1758,27 @@ class Suite(object): def __repr__(self): return '' % self.suite_name + def __eq__(self, val): + if isinstance(val, str): + return (self.suite_name == val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.suite_name != val) + # This signals to use the normal comparison operator + return NotImplemented + + def details(self): + ret = [] + for disp, field in SUITE_FIELDS: + val = getattr(self, field, None) + if val is not None: + ret.append("%s: %s" % (disp, val)) + + return "\n".join(ret) + __all__.append('Suite') def get_suite_architecture(suite, architecture, session=None): @@ -908,13 +1847,21 @@ class SuiteArchitecture(object): __all__.append('SuiteArchitecture') -def get_suite_architectures(suite, session=None): +def get_suite_architectures(suite, skipsrc=False, skipall=False, session=None): """ Returns list of Architecture objects for given C{suite} name @type source: str @param source: Suite name to search for + @type skipsrc: boolean + @param skipsrc: Whether to skip returning the 'source' architecture entry + (Default False) + + @type skipall: boolean + @param skipall: Whether to skip returning the 'all' architecture entry + (Default False) + @type session: Session @param session: Optional SQL session object (a temporary one will be generated if not supplied) @@ -928,7 +1875,12 @@ def get_suite_architectures(suite, session=None): q = session.query(Architecture) q = q.join(SuiteArchitecture) - q = q.join(Suite).filter_by(suite_name=suite).order_by('arch_string') + q = q.join(Suite).filter_by(suite_name=suite) + if skipsrc: + q = q.filter(Architecture.arch_string != 'source') + if skipall: + q = q.filter(Architecture.arch_string != 'all') + q = q.order_by('arch_string') return q.all() __all__.append('get_suite_architectures') @@ -939,11 +1891,110 @@ class Uid(object): def __init__(self, *args, **kwargs): pass + def __eq__(self, val): + if isinstance(val, str): + return (self.uid == val) + # This signals to use the normal comparison operator + return NotImplemented + + def __ne__(self, val): + if isinstance(val, str): + return (self.uid != val) + # This signals to use the normal comparison operator + return NotImplemented + def __repr__(self): return '' % (self.uid, self.name) __all__.append('Uid') +def add_database_user(uidname, session=None): + """ + Adds a database user + + @type uidname: string + @param uidname: The uid of the user to add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. + + @rtype: Uid + @return: the uid object for the given uidname + """ + privatetrans = False + if session is None: + session = DBConn().session() + privatetrans = True + + try: + session.execute("CREATE USER :uid", {'uid': uidname}) + if privatetrans: + session.commit() + except: + traceback.print_exc() + raise + +__all__.append('add_database_user') + +def get_or_set_uid(uidname, session=None): + """ + Returns uid object for given uidname. + + If no matching uidname is found, a row is inserted. + + @type uidname: string + @param uidname: The uid to add + + @type session: SQLAlchemy + @param session: Optional SQL session object (a temporary one will be + generated if not supplied). If not passed, a commit will be performed at + the end of the function, otherwise the caller is responsible for commiting. + + @rtype: Uid + @return: the uid object for the given uidname + """ + privatetrans = False + if session is None: + session = DBConn().session() + privatetrans = True + + try: + q = session.query(Uid).filter_by(uid=uidname) + if q.count() < 1: + uid = Uid() + uid.uid = uidname + session.add(uid) + if privatetrans: + session.commit() + else: + session.flush() + return uid + else: + return q.one() + + except: + traceback.print_exc() + raise + +__all__.append('get_or_set_uid') + + +def get_uid_from_fingerprint(fpr, session=None): + if session is None: + session = DBConn().session() + + q = session.query(Uid) + q = q.join(Fingerprint).filter_by(fingerprint=fpr) + + if q.count() != 1: + return None + else: + return q.one() + +__all__.append('get_uid_from_fingerprint') + ################################################################################ class DBConn(Singleton): @@ -975,6 +2026,7 @@ class DBConn(Singleton): self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) self.tbl_location = Table('location', self.db_meta, autoload=True) self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) + self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) self.tbl_override = Table('override', self.db_meta, autoload=True) self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True) @@ -1002,16 +2054,16 @@ class DBConn(Singleton): suite_id = self.tbl_bin_associations.c.suite, suite = relation(Suite), binary_id = self.tbl_bin_associations.c.bin, - binary = relation(Binary))) + binary = relation(DBBinary))) - mapper(Binary, self.tbl_binaries, + mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, package = self.tbl_binaries.c.package, version = self.tbl_binaries.c.version, maintainer_id = self.tbl_binaries.c.maintainer, maintainer = relation(Maintainer), source_id = self.tbl_binaries.c.source, - source = relation(Source), + source = relation(DBSource), arch_id = self.tbl_binaries.c.architecture, architecture = relation(Architecture), poolfile_id = self.tbl_binaries.c.file, @@ -1037,7 +2089,7 @@ class DBConn(Singleton): filepath_id = self.tbl_content_associations.c.filepath, filepath = relation(ContentFilepath), binary_id = self.tbl_content_associations.c.binary_pkg, - binary = relation(Binary))) + binary = relation(DBBinary))) mapper(ContentFilename, self.tbl_content_file_names, @@ -1051,7 +2103,7 @@ class DBConn(Singleton): mapper(DSCFile, self.tbl_dsc_files, properties = dict(dscfile_id = self.tbl_dsc_files.c.id, source_id = self.tbl_dsc_files.c.source, - source = relation(Source), + source = relation(DBSource), poolfile_id = self.tbl_dsc_files.c.file, poolfile = relation(PoolFile))) @@ -1083,6 +2135,9 @@ class DBConn(Singleton): mapper(Maintainer, self.tbl_maintainer, properties = dict(maintainer_id = self.tbl_maintainer.c.id)) + mapper(NewComment, self.tbl_new_comments, + properties = dict(comment_id = self.tbl_new_comments.c.id)) + mapper(Override, self.tbl_override, properties = dict(suite_id = self.tbl_override.c.suite, suite = relation(Suite), @@ -1115,12 +2170,12 @@ class DBConn(Singleton): mapper(QueueBuild, self.tbl_queue_build, properties = dict(suite_id = self.tbl_queue_build.c.suite, queue_id = self.tbl_queue_build.c.queue, - queue = relation(Queue))) + queue = relation(Queue, backref='queuebuild'))) mapper(Section, self.tbl_section, properties = dict(section_id = self.tbl_section.c.id)) - mapper(Source, self.tbl_source, + mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, version = self.tbl_source.c.version, maintainer_id = self.tbl_source.c.maintainer, @@ -1143,12 +2198,12 @@ class DBConn(Singleton): suite_id = self.tbl_src_associations.c.suite, suite = relation(Suite), source_id = self.tbl_src_associations.c.source, - source = relation(Source))) + source = relation(DBSource))) mapper(SrcUploader, self.tbl_src_uploaders, properties = dict(uploader_id = self.tbl_src_uploaders.c.id, source_id = self.tbl_src_uploaders.c.source, - source = relation(Source, + source = relation(DBSource, primaryjoin=(self.tbl_src_uploaders.c.source==self.tbl_source.c.id)), maintainer_id = self.tbl_src_uploaders.c.maintainer, maintainer = relation(Maintainer, @@ -1159,12 +2214,13 @@ class DBConn(Singleton): mapper(SuiteArchitecture, self.tbl_suite_architectures, properties = dict(suite_id = self.tbl_suite_architectures.c.suite, - suite = relation(Suite), + suite = relation(Suite, backref='suitearchitectures'), arch_id = self.tbl_suite_architectures.c.architecture, architecture = relation(Architecture))) mapper(Uid, self.tbl_uid, - properties = dict(uid_id = self.tbl_uid.c.id)) + properties = dict(uid_id = self.tbl_uid.c.id, + fingerprint = relation(Fingerprint))) ## Connection functions def __createconn(self): @@ -1197,3 +2253,4 @@ class DBConn(Singleton): __all__.append('DBConn') +