X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fdbconn.py;h=84d5b1c2c059fd586793bdbe54d1b0f0468a95d6;hb=881011792a47f49d6fe2354a7efb86eee6bb572b;hp=779c0e2936a3fa270fe68b290c22b9d614d3f2d6;hpb=b96fec590c5123bd91db1144ae813e2a133f3571;p=dak.git diff --git a/daklib/dbconn.py b/daklib/dbconn.py old mode 100755 new mode 100644 index 779c0e29..84d5b1c2 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -37,7 +37,9 @@ import os import re import psycopg2 import traceback -from datetime import datetime +from datetime import datetime, timedelta +from errno import ENOENT +from tempfile import mkstemp, mkdtemp from inspect import getargspec @@ -51,7 +53,6 @@ from sqlalchemy.exc import * from sqlalchemy.orm.exc import NoResultFound from config import Config -from singleton import Singleton from textutils import fix_maintainer ################################################################################ @@ -430,12 +431,188 @@ __all__.append('BinaryACLMap') ################################################################################ +MINIMAL_APT_CONF=""" +Dir +{ + ArchiveDir "%(archivepath)s"; + OverrideDir "/srv/ftp.debian.org/scripts/override/"; + CacheDir "/srv/ftp.debian.org/database/"; +}; + +Default +{ + Packages::Compress ". bzip2 gzip"; + Sources::Compress ". bzip2 gzip"; + DeLinkLimit 0; + FileMode 0664; +} + +bindirectory "incoming" +{ + Packages "Packages"; + Contents " "; + + BinOverride "override.sid.all3"; + BinCacheDB "packages-accepted.db"; + + FileList "%(filelist)s"; + + PathPrefix ""; + Packages::Extensions ".deb .udeb"; +}; + +bindirectory "incoming/" +{ + Sources "Sources"; + BinOverride "override.sid.all3"; + SrcOverride "override.sid.all3.src"; + FileList "%(filelist)s"; +}; +""" + class BuildQueue(object): def __init__(self, *args, **kwargs): pass def __repr__(self): - return '' % self.queue_name + return '' % self.queue_name + + def write_metadata(self, starttime, force=False): + # Do we write out metafiles? + if not (force or self.generate_metadata): + return + + session = DBConn().session().object_session(self) + + fl_fd = fl_name = ac_fd = ac_name = None + tempdir = None + arches = " ".join([ a.arch_string for a in session.query(Architecture).all() if a.arch_string != 'source' ]) + startdir = os.getcwd() + + try: + # Grab files we want to include + newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all() + # Write file list with newer files + (fl_fd, fl_name) = mkstemp() + for n in newer: + os.write(fl_fd, '%s\n' % n.fullpath) + os.close(fl_fd) + + # Write minimal apt.conf + # TODO: Remove hardcoding from template + (ac_fd, ac_name) = mkstemp() + os.write(ac_fd, MINIMAL_APT_CONF % {'archivepath': self.path, + 'filelist': fl_name}) + os.close(ac_fd) + + # Run apt-ftparchive generate + os.chdir(os.path.dirname(ac_name)) + os.system('apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate %s' % os.path.basename(ac_name)) + + # Run apt-ftparchive release + # TODO: Eww - fix this + bname = os.path.basename(self.path) + os.chdir(self.path) + os.chdir('..') + + # We have to remove the Release file otherwise it'll be included in the + # new one + try: + os.unlink(os.path.join(bname, 'Release')) + except OSError: + pass + + os.system("""apt-ftparchive -qq -o APT::FTPArchive::Release::Origin="%s" -o APT::FTPArchive::Release::Label="%s" -o APT::FTPArchive::Release::Description="%s" -o APT::FTPArchive::Release::Architectures="%s" release %s > Release""" % (self.origin, self.label, self.releasedescription, arches, bname)) + + # Sign if necessary + if self.signingkey: + cnf = Config() + keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"] + if cnf.has_key("Dinstall::SigningPubKeyring"): + keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"] + + os.system("gpg %s --no-options --batch --no-tty --armour --default-key %s --detach-sign -o Release.gpg Release""" % (keyring, self.signingkey)) + + # Move the files if we got this far + os.rename('Release', os.path.join(bname, 'Release')) + if self.signingkey: + os.rename('Release.gpg', os.path.join(bname, 'Release.gpg')) + + # Clean up any left behind files + finally: + os.chdir(startdir) + if fl_fd: + try: + os.close(fl_fd) + except OSError: + pass + + if fl_name: + try: + os.unlink(fl_name) + except OSError: + pass + + if ac_fd: + try: + os.close(ac_fd) + except OSError: + pass + + if ac_name: + try: + os.unlink(ac_name) + except OSError: + pass + + def clean_and_update(self, starttime, Logger, dryrun=False): + """WARNING: This routine commits for you""" + session = DBConn().session().object_session(self) + + if self.generate_metadata and not dryrun: + self.write_metadata(starttime) + + # Grab files older than our execution time + older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all() + + for o in older: + killdb = False + try: + if dryrun: + Logger.log(["I: Would have removed %s from the queue" % o.fullpath]) + else: + Logger.log(["I: Removing %s from the queue" % o.fullpath]) + os.unlink(o.fullpath) + killdb = True + except OSError, e: + # If it wasn't there, don't worry + if e.errno == ENOENT: + killdb = True + else: + # TODO: Replace with proper logging call + Logger.log(["E: Could not remove %s" % o.fullpath]) + + if killdb: + session.delete(o) + + session.commit() + + for f in os.listdir(self.path): + if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release'): + continue + + try: + r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one() + except NoResultFound: + fp = os.path.join(self.path, f) + if dryrun: + Logger.log(["I: Would remove unused link %s" % fp]) + else: + Logger.log(["I: Removing unused link %s" % fp]) + try: + os.unlink(fp) + except OSError: + Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath]) def add_file_from_pool(self, poolfile): """Copies a file into the pool. Assumes that the PoolFile object is @@ -448,30 +625,30 @@ class BuildQueue(object): for f in self.queuefiles: if f.fileid is not None and f.fileid == poolfile.file_id or \ f.poolfile.filename == poolfile_basename: - # In this case, update the QueueFile entry so we + # In this case, update the BuildQueueFile entry so we # don't remove it too early f.lastused = datetime.now() - DBConn().session().object_session(pf).add(f) + DBConn().session().object_session(poolfile).add(f) return f - # Prepare QueueFile object - qf = QueueFile() - qf.queue_id = self.queue_id + # Prepare BuildQueueFile object + qf = BuildQueueFile() + qf.build_queue_id = self.queue_id qf.lastused = datetime.now() - qf.filename = dest + qf.filename = poolfile_basename - targetpath = qf.fullpath + targetpath = poolfile.fullpath queuepath = os.path.join(self.path, poolfile_basename) try: - if self.copy_pool_files: + if self.copy_files: # We need to copy instead of symlink import utils - utils.copy(targetfile, queuepath) + utils.copy(targetpath, queuepath) # NULL in the fileid field implies a copy qf.fileid = None else: - os.symlink(targetfile, queuepath) + os.symlink(targetpath, queuepath) qf.fileid = poolfile.file_id except OSError: return None @@ -485,9 +662,9 @@ class BuildQueue(object): __all__.append('BuildQueue') @session_wrapper -def get_queue(queuename, session=None): +def get_build_queue(queuename, session=None): """ - Returns Queue object for given C{queue name}, creating it if it does not + Returns BuildQueue object for given C{queue name}, creating it if it does not exist. @type queuename: string @@ -497,18 +674,18 @@ def get_queue(queuename, session=None): @param session: Optional SQLA session object (a temporary one will be generated if not supplied) - @rtype: Queue - @return: Queue object for the given queue + @rtype: BuildQueue + @return: BuildQueue object for the given queue """ - q = session.query(Queue).filter_by(queue_name=queuename) + q = session.query(BuildQueue).filter_by(queue_name=queuename) try: return q.one() except NoResultFound: return None -__all__.append('get_queue') +__all__.append('get_build_queue') ################################################################################ @@ -517,7 +694,12 @@ class BuildQueueFile(object): pass def __repr__(self): - return '' % (self.filename, self.queue_id) + return '' % (self.filename, self.build_queue_id) + + @property + def fullpath(self): + return os.path.join(self.buildqueue.path, self.filename) + __all__.append('BuildQueueFile') @@ -1659,6 +1841,31 @@ class PolicyQueue(object): __all__.append('PolicyQueue') +@session_wrapper +def get_policy_queue(queuename, session=None): + """ + Returns PolicyQueue object for given C{queue name} + + @type queuename: string + @param queuename: The name of the queue + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: PolicyQueue + @return: PolicyQueue object for the given queue + """ + + q = session.query(PolicyQueue).filter_by(queue_name=queuename) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_policy_queue') + ################################################################################ class Priority(object): @@ -1965,6 +2172,7 @@ __all__.append('get_source_in_suite') def add_dsc_to_db(u, filename, session=None): entry = u.pkg.files[filename] source = DBSource() + pfs = [] source.source = u.pkg.dsc["source"] source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch @@ -1983,6 +2191,7 @@ def add_dsc_to_db(u, filename, session=None): filename = entry["pool name"] + filename poolfile = add_poolfile(filename, entry, dsc_location_id, session) session.flush() + pfs.append(poolfile) entry["files id"] = poolfile.file_id source.poolfile_id = entry["files id"] @@ -2026,6 +2235,7 @@ def add_dsc_to_db(u, filename, session=None): # FIXME: needs to check for -1/-2 and or handle exception if found and obj is not None: files_id = obj.file_id + pfs.append(obj) # If still not found, add it if files_id is None: @@ -2033,7 +2243,13 @@ def add_dsc_to_db(u, filename, session=None): dentry["sha1sum"] = dfentry["sha1sum"] dentry["sha256sum"] = dfentry["sha256sum"] poolfile = add_poolfile(filename, dentry, dsc_location_id, session) + pfs.append(poolfile) files_id = poolfile.file_id + else: + poolfile = get_poolfile_by_id(files_id, session) + if poolfile is None: + utils.fubar("INTERNAL ERROR. Found no poolfile with id %d" % files_id) + pfs.append(poolfile) df.poolfile_id = files_id session.add(df) @@ -2062,7 +2278,7 @@ def add_dsc_to_db(u, filename, session=None): session.flush() - return dsc_component, dsc_location_id + return dsc_component, dsc_location_id, pfs __all__.append('add_dsc_to_db') @@ -2090,11 +2306,12 @@ def add_deb_to_db(u, filename, session=None): if not entry.get("location id", None): entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id - if not entry.get("files id", None): + if entry.get("files id", None): + poolfile = get_poolfile_by_id(bin.poolfile_id) + bin.poolfile_id = entry["files id"] + else: poolfile = add_poolfile(filename, entry, entry["location id"], session) - entry["files id"] = poolfile.file_id - - bin.poolfile_id = entry["files id"] + bin.poolfile_id = entry["files id"] = poolfile.file_id # Find source id bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session) @@ -2125,6 +2342,8 @@ def add_deb_to_db(u, filename, session=None): # session.rollback() # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename) + return poolfile + __all__.append('add_deb_to_db') ################################################################################ @@ -2470,65 +2689,72 @@ __all__.append('UploadBlock') ################################################################################ -class DBConn(Singleton): +class DBConn(object): """ database module init. """ + __shared_state = {} + def __init__(self, *args, **kwargs): - super(DBConn, self).__init__(*args, **kwargs) + self.__dict__ = self.__shared_state - def _startup(self, *args, **kwargs): - self.debug = False - if kwargs.has_key('debug'): - self.debug = True - self.__createconn() + if not getattr(self, 'initialised', False): + self.initialised = True + self.debug = kwargs.has_key('debug') + self.__createconn() def __setuptables(self): - self.tbl_architecture = Table('architecture', self.db_meta, autoload=True) - self.tbl_archive = Table('archive', self.db_meta, autoload=True) - self.tbl_bin_associations = Table('bin_associations', self.db_meta, autoload=True) - self.tbl_binaries = Table('binaries', self.db_meta, autoload=True) - self.tbl_binary_acl = Table('binary_acl', self.db_meta, autoload=True) - self.tbl_binary_acl_map = Table('binary_acl_map', self.db_meta, autoload=True) - self.tbl_build_queue = Table('build_queue', self.db_meta, autoload=True) - self.tbl_build_queue_files = Table('build_queue_files', self.db_meta, autoload=True) - self.tbl_component = Table('component', self.db_meta, autoload=True) - self.tbl_config = Table('config', self.db_meta, autoload=True) - self.tbl_content_associations = Table('content_associations', self.db_meta, autoload=True) - self.tbl_content_file_names = Table('content_file_names', self.db_meta, autoload=True) - self.tbl_content_file_paths = Table('content_file_paths', self.db_meta, autoload=True) - self.tbl_changes_pending_binary = Table('changes_pending_binaries', self.db_meta, autoload=True) - self.tbl_changes_pending_files = Table('changes_pending_files', self.db_meta, autoload=True) - self.tbl_changes_pending_files_map = Table('changes_pending_files_map', self.db_meta, autoload=True) - self.tbl_changes_pending_source = Table('changes_pending_source', self.db_meta, autoload=True) - self.tbl_changes_pending_source_files = Table('changes_pending_source_files', self.db_meta, autoload=True) - self.tbl_changes_pool_files = Table('changes_pool_files', self.db_meta, autoload=True) - self.tbl_dsc_files = Table('dsc_files', self.db_meta, autoload=True) - self.tbl_files = Table('files', self.db_meta, autoload=True) - self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) - self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) - self.tbl_changes = Table('changes', self.db_meta, autoload=True) - self.tbl_keyring_acl_map = Table('keyring_acl_map', self.db_meta, autoload=True) - self.tbl_location = Table('location', self.db_meta, autoload=True) - self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) - self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) - self.tbl_override = Table('override', self.db_meta, autoload=True) - self.tbl_override_type = Table('override_type', self.db_meta, autoload=True) - self.tbl_pending_content_associations = Table('pending_content_associations', self.db_meta, autoload=True) - self.tbl_policy_queue = Table('policy_queue', self.db_meta, autoload=True) - self.tbl_priority = Table('priority', self.db_meta, autoload=True) - self.tbl_section = Table('section', self.db_meta, autoload=True) - self.tbl_source = Table('source', self.db_meta, autoload=True) - self.tbl_source_acl = Table('source_acl', self.db_meta, autoload=True) - self.tbl_src_associations = Table('src_associations', self.db_meta, autoload=True) - self.tbl_src_format = Table('src_format', self.db_meta, autoload=True) - self.tbl_src_uploaders = Table('src_uploaders', self.db_meta, autoload=True) - self.tbl_suite = Table('suite', self.db_meta, autoload=True) - self.tbl_suite_architectures = Table('suite_architectures', self.db_meta, autoload=True) - self.tbl_suite_src_formats = Table('suite_src_formats', self.db_meta, autoload=True) - self.tbl_suite_build_queue_copy = Table('suite_build_queue_copy', self.db_meta, autoload=True) - self.tbl_uid = Table('uid', self.db_meta, autoload=True) - self.tbl_upload_blocks = Table('upload_blocks', self.db_meta, autoload=True) + tables = ( + 'architecture', + 'archive', + 'bin_associations', + 'binaries', + 'binary_acl', + 'binary_acl_map', + 'build_queue', + 'build_queue_files', + 'component', + 'config', + 'content_associations', + 'content_file_names', + 'content_file_paths', + 'changes_pending_binaries', + 'changes_pending_files', + 'changes_pending_files_map', + 'changes_pending_source', + 'changes_pending_source_files', + 'changes_pool_files', + 'dsc_files', + 'files', + 'fingerprint', + 'keyrings', + 'changes', + 'keyring_acl_map', + 'location', + 'maintainer', + 'new_comments', + 'override', + 'override_type', + 'pending_content_associations', + 'policy_queue', + 'priority', + 'section', + 'source', + 'source_acl', + 'src_associations', + 'src_format', + 'src_uploaders', + 'suite', + 'suite_architectures', + 'suite_src_formats', + 'suite_build_queue_copy', + 'uid', + 'upload_blocks', + ) + + for table_name in tables: + table = Table(table_name, self.db_meta, autoload=True) + setattr(self, 'tbl_%s' % table_name, table) def __setupmappers(self): mapper(Architecture, self.tbl_architecture, @@ -2625,8 +2851,8 @@ class DBConn(Singleton): primaryjoin=(self.tbl_changes.c.in_queue==self.tbl_policy_queue.c.id)), approved_for_id = self.tbl_changes.c.approved_for)) - mapper(ChangePendingBinary, self.tbl_changes_pending_binary, - properties = dict(change_pending_binary_id = self.tbl_changes_pending_binary.c.id)) + mapper(ChangePendingBinary, self.tbl_changes_pending_binaries, + properties = dict(change_pending_binary_id = self.tbl_changes_pending_binaries.c.id)) mapper(ChangePendingFile, self.tbl_changes_pending_files, properties = dict(change_pending_file_id = self.tbl_changes_pending_files.c.id))