X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=921f1daa03af8c35e066c777ba8872ebd4b5bba8;hb=3b50b545815298b77b8eb68930acb6fde01ea4d4;hp=75d271bd39d1cbff799fa9b02d68cfa8345b6f5d;hpb=6021a91078574c97968d076c0320182a20c33649;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 75d271bd..921f1daa 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -37,6 +37,7 @@ import os import re import psycopg2 import traceback +import datetime from inspect import getargspec @@ -66,10 +67,10 @@ class DebVersion(sqltypes.Text): sa_major_version = sqlalchemy.__version__[0:3] if sa_major_version == "0.5": - from sqlalchemy.databases import postgres - postgres.ischema_names['debversion'] = DebVersion + from sqlalchemy.databases import postgres + postgres.ischema_names['debversion'] = DebVersion else: - raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py") + raise Exception("dak isn't ported to SQLA versions != 0.5 yet. See daklib/dbconn.py") ################################################################################ @@ -286,12 +287,12 @@ def get_suites_binary_in(package, session=None): __all__.append('get_suites_binary_in') @session_wrapper -def get_binary_from_id(id, session=None): +def get_binary_from_id(binary_id, session=None): """ Returns DBBinary object for given C{id} - @type id: int - @param id: Id of the required binary + @type binary_id: int + @param binary_id: Id of the required binary @type session: Session @param session: Optional SQLA session object (a temporary one will be @@ -301,7 +302,7 @@ def get_binary_from_id(id, session=None): @return: DBBinary object for the given binary (None if not present) """ - q = session.query(DBBinary).filter_by(binary_id=id) + q = session.query(DBBinary).filter_by(binary_id=binary_id) try: return q.one() @@ -652,11 +653,16 @@ def insert_content_paths(binary_id, fullpaths, session=None): try: # Insert paths pathcache = {} - for fullpath in fullpaths: - if fullpath.startswith( './' ): - fullpath = fullpath[2:] - session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", { 'filename': fullpath, 'id': binary_id} ) + def generate_path_dicts(): + for fullpath in fullpaths: + if fullpath.startswith( './' ): + fullpath = fullpath[2:] + + yield {'fulename':fullpath, 'id': binary_id } + + session.execute( "INSERT INTO bin_contents ( file, binary_id ) VALUES ( :filename, :id )", + generate_path_dicts() ) session.commit() if privatetrans: @@ -728,6 +734,10 @@ class PoolFile(object): def __repr__(self): return '' % self.filename + @property + def fullpath(self): + return os.path.join(self.location.path, self.filename) + __all__.append('PoolFile') @session_wrapper @@ -769,7 +779,7 @@ def check_poolfile(filename, filesize, md5sum, location_id, session=None): ret = (False, None) else: obj = q.one() - if obj.md5sum != md5sum or obj.filesize != filesize: + if obj.md5sum != md5sum or obj.filesize != int(filesize): ret = (False, obj) if ret is None: @@ -944,8 +954,8 @@ class Keyring(object): def __repr__(self): return '' % self.keyring_name - def de_escape_gpg_str(self, str): - esclist = re.split(r'(\\x..)', str) + def de_escape_gpg_str(self, txt): + esclist = re.split(r'(\\x..)', txt) for x in range(1,len(esclist),2): esclist[x] = "%c" % (int(esclist[x][2:],16)) return "".join(esclist) @@ -1119,6 +1129,18 @@ def get_knownchange(filename, session=None): __all__.append('get_knownchange') ################################################################################ + +class KnownChangePendingFile(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.known_change_pending_file_id + +__all__.append('KnownChangePendingFile') + +################################################################################ + class Location(object): def __init__(self, *args, **kwargs): pass @@ -1397,16 +1419,38 @@ __all__.append('get_override_type') ################################################################################ -class PendingContentAssociation(object): +class DebContents(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.pca_id + return '' % (self.package.package,self.file) -__all__.append('PendingContentAssociation') +__all__.append('DebContents') + + +class UdebContents(object): + def __init__(self, *args, **kwargs): + pass -def insert_pending_content_paths(package, fullpaths, session=None): + def __repr__(self): + return '' % (self.package.package,self.file) + +__all__.append('UdebContents') + +class PendingBinContents(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.contents_id + +__all__.append('PendingBinContents') + +def insert_pending_content_paths(package, + is_udeb, + fullpaths, + session=None): """ Make sure given paths are temporarily associated with given package @@ -1435,32 +1479,27 @@ def insert_pending_content_paths(package, fullpaths, session=None): arch_id = arch.arch_id # Remove any already existing recorded files for this package - q = session.query(PendingContentAssociation) + q = session.query(PendingBinContents) q = q.filter_by(package=package['Package']) q = q.filter_by(version=package['Version']) q = q.filter_by(architecture=arch_id) q.delete() - # Insert paths - pathcache = {} for fullpath in fullpaths: - (path, file) = os.path.split(fullpath) - - if path.startswith( "./" ): - path = path[2:] - filepath_id = get_or_set_contents_path_id(path, session) - filename_id = get_or_set_contents_file_id(file, session) - - pathcache[fullpath] = (filepath_id, filename_id) + if fullpath.startswith( "./" ): + fullpath = fullpath[2:] - for fullpath, dat in pathcache.items(): - pca = PendingContentAssociation() + pca = PendingBinContents() pca.package = package['Package'] pca.version = package['Version'] - pca.filepath_id = dat[0] - pca.filename_id = dat[1] + pca.file = fullpath pca.architecture = arch_id + + if isudeb: + pca.type = 8 # gross + else: + pca.type = 7 # also gross session.add(pca) # Only commit if we set up the session ourself @@ -1562,127 +1601,55 @@ class Queue(object): def __repr__(self): return '' % self.queue_name - def autobuild_upload(self, changes, srcpath, session=None): - """ - Update queue_build database table used for incoming autobuild support. + def add_file_from_pool(self, poolfile): + """Copies a file into the pool. Assumes that the PoolFile object is + attached to the same SQLAlchemy session as the Queue object is. - @type changes: Changes - @param changes: changes object for the upload to process + The caller is responsible for committing after calling this function.""" + poolfile_basename = poolfile.filename[poolfile.filename.rindex(os.sep)+1:] - @type srcpath: string - @param srcpath: path for the queue file entries/link destinations + # Check if we have a file of this name or this ID already + for f in self.queuefiles: + if f.fileid is not None and f.fileid == poolfile.file_id or \ + f.poolfile.filename == poolfile_basename: + # In this case, update the QueueFile entry so we + # don't remove it too early + f.lastused = datetime.now() + DBConn().session().object_session(pf).add(f) + return f - @type session: SQLAlchemy session - @param session: Optional SQLAlchemy session. If this is passed, the - caller is responsible for ensuring a transaction has begun and - committing the results or rolling back based on the result code. If - not passed, a commit will be performed at the end of the function, - otherwise the caller is responsible for commiting. + # Prepare QueueFile object + qf = QueueFile() + qf.queue_id = self.queue_id + qf.lastused = datetime.now() + qf.filename = dest - @rtype: NoneType or string - @return: None if the operation failed, a string describing the error if not - """ + targetpath = qf.fullpath + queuepath = os.path.join(self.path, poolfile_basename) - privatetrans = False - if session is None: - session = DBConn().session() - privatetrans = True - - # TODO: Remove by moving queue config into the database - conf = Config() - - for suitename in changes.changes["distribution"].keys(): - # TODO: Move into database as: - # buildqueuedir TEXT DEFAULT NULL (i.e. NULL is no build) - # buildqueuecopy BOOLEAN NOT NULL DEFAULT FALSE (i.e. default is symlink) - # This also gets rid of the SecurityQueueBuild hack below - if suitename not in conf.ValueList("Dinstall::QueueBuildSuites"): - continue - - # Find suite object - s = get_suite(suitename, session) - if s is None: - return "INTERNAL ERROR: Could not find suite %s" % suitename - - # TODO: Get from database as above - dest_dir = conf["Dir::QueueBuild"] - - # TODO: Move into database as above - if conf.FindB("Dinstall::SecurityQueueBuild"): - dest_dir = os.path.join(dest_dir, suitename) - - for file_entry in changes.files.keys(): - src = os.path.join(srcpath, file_entry) - dest = os.path.join(dest_dir, file_entry) - - # TODO: Move into database as above - if conf.FindB("Dinstall::SecurityQueueBuild"): - # Copy it since the original won't be readable by www-data - import utils - utils.copy(src, dest) - else: - # Create a symlink to it - os.symlink(src, dest) - - qb = QueueBuild() - qb.suite_id = s.suite_id - qb.queue_id = self.queue_id - qb.filename = dest - qb.in_queue = True - - session.add(qb) - - # If the .orig tarballs are in the pool, create a symlink to - # them (if one doesn't already exist) - for dsc_file in changes.dsc_files.keys(): - # Skip all files except orig tarballs - from daklib.regexes import re_is_orig_source - if not re_is_orig_source.match(dsc_file): - continue - # Skip orig files not identified in the pool - if not (changes.orig_files.has_key(dsc_file) and - changes.orig_files[dsc_file].has_key("id")): - continue - orig_file_id = changes.orig_files[dsc_file]["id"] - dest = os.path.join(dest_dir, dsc_file) - - # If it doesn't exist, create a symlink - if not os.path.exists(dest): - q = session.execute("SELECT l.path, f.filename FROM location l, files f WHERE f.id = :id and f.location = l.id", - {'id': orig_file_id}) - res = q.fetchone() - if not res: - return "[INTERNAL ERROR] Couldn't find id %s in files table." % (orig_file_id) - - src = os.path.join(res[0], res[1]) - os.symlink(src, dest) - - # Add it to the list of packages for later processing by apt-ftparchive - qb = QueueBuild() - qb.suite_id = s.suite_id - qb.queue_id = self.queue_id - qb.filename = dest - qb.in_queue = True - session.add(qb) - - # If it does, update things to ensure it's not removed prematurely - else: - qb = get_queue_build(dest, s.suite_id, session) - if qb is None: - qb.in_queue = True - qb.last_used = None - session.add(qb) + try: + if self.copy_pool_files: + # We need to copy instead of symlink + import utils + utils.copy(targetfile, queuepath) + # NULL in the fileid field implies a copy + qf.fileid = None + else: + os.symlink(targetfile, queuepath) + qf.fileid = poolfile.file_id + except OSError: + return None - if privatetrans: - session.commit() - session.close() + # Get the same session as the PoolFile is using and add the qf to it + DBConn().session().object_session(poolfile).add(qf) + + return qf - return None __all__.append('Queue') @session_wrapper -def get_or_set_queue(queuename, session=None): +def get_queue(queuename, session=None): """ Returns Queue object for given C{queue name}, creating it if it does not exist. @@ -1701,60 +1668,22 @@ def get_or_set_queue(queuename, session=None): q = session.query(Queue).filter_by(queue_name=queuename) try: - ret = q.one() + return q.one() except NoResultFound: - queue = Queue() - queue.queue_name = queuename - session.add(queue) - session.commit_or_flush() - ret = queue - - return ret + return None -__all__.append('get_or_set_queue') +__all__.append('get_queue') ################################################################################ -class QueueBuild(object): +class QueueFile(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % (self.filename, self.queue_id) - -__all__.append('QueueBuild') + return '' % (self.filename, self.queue_id) -@session_wrapper -def get_queue_build(filename, suite, session=None): - """ - Returns QueueBuild object for given C{filename} and C{suite}. - - @type filename: string - @param filename: The name of the file - - @type suiteid: int or str - @param suiteid: Suite name or ID - - @type session: Session - @param session: Optional SQLA session object (a temporary one will be - generated if not supplied) - - @rtype: Queue - @return: Queue object for the given queue - """ - - if isinstance(suite, int): - q = session.query(QueueBuild).filter_by(filename=filename).filter_by(suite_id=suite) - else: - q = session.query(QueueBuild).filter_by(filename=filename) - q = q.join(Suite).filter_by(suite_name=suite) - - try: - return q.one() - except NoResultFound: - return None - -__all__.append('get_queue_build') +__all__.append('QueueFile') ################################################################################ @@ -2345,6 +2274,7 @@ class DBConn(Singleton): def __setuptables(self): self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) self.tbl_archive = Table('archive', self.db_meta, autoload=True) + self.tbl_bin_contents = Table('bin_contents', self.db_meta, autoload=True) self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) @@ -2354,7 +2284,10 @@ class DBConn(Singleton): self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True) self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True) + self.tbl_changes_pending_files = Table('changes_pending_files', self.db_meta, autoload=True) + self.tbl_changes_pool_files = Table('changes_pool_files', self.db_meta, autoload=True) self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True) + self.tbl_deb_contents = Table('deb_contents', self.db_meta, autoload=True) self.tbl_files = Table('files', self.db_meta, autoload=True) self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) @@ -2365,10 +2298,10 @@ class DBConn(Singleton): self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) self.tbl_override = Table('override', self.db_meta, autoload=True) self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) - self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True) + self.tbl_pending_bin_contents = Table('pending_bin_contents', self.db_meta, autoload=True) self.tbl_priority = Table('priority', self.db_meta, autoload=True) self.tbl_queue = Table('queue', self.db_meta, autoload=True) - self.tbl_queue_build = Table('queue_build', self.db_meta, autoload=True) + self.tbl_queue_files = Table('queue_files', self.db_meta, autoload=True) self.tbl_section = Table('section', self.db_meta, autoload=True) self.tbl_source = Table('source', self.db_meta, autoload=True) self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) @@ -2378,6 +2311,8 @@ class DBConn(Singleton): self.tbl_suite = Table('suite', self.db_meta, autoload=True) self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) + self.tbl_suite_queue_copy = Table('suite_queue_copy', self.db_meta, autoload=True) + self.tbl_udeb_contents = Table('udeb_contents', self.db_meta, autoload=True) self.tbl_uid = Table('uid', self.db_meta, autoload=True) self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) @@ -2396,6 +2331,29 @@ class DBConn(Singleton): binary_id = self.tbl_bin_associations.c.bin, binary = relation(DBBinary))) + mapper(PendingBinContents, self.tbl_pending_bin_contents, + properties = dict(contents_id =self.tbl_pending_bin_contents.c.id, + filename = self.tbl_pending_bin_contents.c.filename, + package = self.tbl_pending_bin_contents.c.package, + version = self.tbl_pending_bin_contents.c.version, + arch = self.tbl_pending_bin_contents.c.arch, + otype = self.tbl_pending_bin_contents.c.type)) + + mapper(DebContents, self.tbl_deb_contents, + properties = dict(binary_id=self.tbl_deb_contents.c.binary_id, + package=self.tbl_deb_contents.c.package, + component=self.tbl_deb_contents.c.component, + arch=self.tbl_deb_contents.c.arch, + section=self.tbl_deb_contents.c.section, + filename=self.tbl_deb_contents.c.filename)) + + mapper(UdebContents, self.tbl_udeb_contents, + properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id, + package=self.tbl_udeb_contents.c.package, + component=self.tbl_udeb_contents.c.component, + arch=self.tbl_udeb_contents.c.arch, + section=self.tbl_udeb_contents.c.section, + filename=self.tbl_udeb_contents.c.filename)) mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, @@ -2458,7 +2416,14 @@ class DBConn(Singleton): keyring_id = self.tbl_keyrings.c.id)) mapper(KnownChange, self.tbl_known_changes, - properties = dict(known_change_id = self.tbl_known_changes.c.id)) + properties = dict(known_change_id = self.tbl_known_changes.c.id, + poolfiles = relation(PoolFile, + secondary=self.tbl_changes_pool_files, + backref="changeslinks"), + files = relation(KnownChangePendingFile, backref="changesfile"))) + + mapper(KnownChangePendingFile, self.tbl_changes_pending_files, + properties = dict(known_change_pending_file_id = self.tbl_changes_pending_files.c.id)) mapper(KeyringACLMap, self.tbl_keyring_acl_map, properties = dict(keyring_acl_map_id = self.tbl_keyring_acl_map.c.id, @@ -2482,6 +2447,7 @@ class DBConn(Singleton): mapper(Override, self.tbl_override, properties = dict(suite_id = self.tbl_override.c.suite, suite = relation(Suite), + package = self.tbl_override.c.package, component_id = self.tbl_override.c.component, component = relation(Component), priority_id = self.tbl_override.c.priority, @@ -2501,13 +2467,13 @@ class DBConn(Singleton): mapper(Queue, self.tbl_queue, properties = dict(queue_id = self.tbl_queue.c.id)) - mapper(QueueBuild, self.tbl_queue_build, - properties = dict(suite_id = self.tbl_queue_build.c.suite, - queue_id = self.tbl_queue_build.c.queue, - queue = relation(Queue, backref='queuebuild'))) + mapper(QueueFile, self.tbl_queue_files, + properties = dict(queue = relation(Queue, backref='queuefiles'), + poolfile = relation(PoolFile, backref='queueinstances'))) mapper(Section, self.tbl_section, - properties = dict(section_id = self.tbl_section.c.id)) + properties = dict(section_id = self.tbl_section.c.id, + section=self.tbl_section.c.section)) mapper(DBSource, self.tbl_source, properties = dict(source_id = self.tbl_source.c.id, @@ -2553,7 +2519,8 @@ class DBConn(Singleton): mapper(Suite, self.tbl_suite, properties = dict(suite_id = self.tbl_suite.c.id, - policy_queue = relation(Queue))) + policy_queue = relation(Queue), + copy_queues = relation(Queue, secondary=self.tbl_suite_queue_copy))) mapper(SuiteArchitecture, self.tbl_suite_architectures, properties = dict(suite_id = self.tbl_suite_architectures.c.suite,