From: Ansgar Burchardt Date: Fri, 25 Mar 2011 13:58:42 +0000 (+0100) Subject: Merge remote-tracking branch 'origin/master' into build-queues X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=6b1a0d77983a7a2dd80630cc46ad659fa4cc3b7a;hp=f8ed5b9335b841bd4fca5b43a3b50fbd0f5baba4;p=dak.git Merge remote-tracking branch 'origin/master' into build-queues --- diff --git a/dak/dakdb/update53.py b/dak/dakdb/update53.py new file mode 100755 index 00000000..36a076f6 --- /dev/null +++ b/dak/dakdb/update53.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add table for build queue files from policy queues. + +@contact: Debian FTP Master +@copyright: 2011 Ansgar Burchardt +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError + +################################################################################ +def do_update(self): + """ + Add table for build queue files from policy queues. + """ + print __doc__ + try: + c = self.db.cursor() + + c.execute(""" + CREATE TABLE build_queue_policy_files ( + build_queue_id INTEGER NOT NULL REFERENCES build_queue(id) ON DELETE CASCADE, + file_id INTEGER NOT NULL REFERENCES changes_pending_files(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + created TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + lastused TIMESTAMP WITHOUT TIME ZONE, + PRIMARY KEY (build_queue_id, file_id) + )""") + + c.execute("UPDATE config SET value = '53' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, 'Unable to apply sick update 53, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/dakdb/update54.py b/dak/dakdb/update54.py new file mode 100755 index 00000000..e7676ef0 --- /dev/null +++ b/dak/dakdb/update54.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Add send_to_build_queues to policy_queue table + +@contact: Debian FTP Master +@copyright: 2011 Ansgar Burchardt +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import psycopg2 +from daklib.dak_exceptions import DBUpdateError + +################################################################################ +def do_update(self): + """ + Add send_to_build_queues to policy_queue table + """ + print __doc__ + try: + c = self.db.cursor() + + c.execute(""" + ALTER TABLE policy_queue ADD COLUMN send_to_build_queues BOOLEAN NOT NULL DEFAULT 'f' + """) + c.execute(""" + UPDATE policy_queue SET send_to_build_queues='t' WHERE queue_name IN ('embargo', 'disembargo') + """) + + c.execute("UPDATE config SET value = '54' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, 'Unable to apply sick update 54, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/update_db.py b/dak/update_db.py index 9051704b..88ff20f5 100755 --- a/dak/update_db.py +++ b/dak/update_db.py @@ -46,7 +46,7 @@ from daklib.daklog import Logger ################################################################################ Cnf = None -required_database_schema = 52 +required_database_schema = 54 ################################################################################ diff --git a/daklib/changes.py b/daklib/changes.py index e016638c..54adb3b0 100644 --- a/daklib/changes.py +++ b/daklib/changes.py @@ -187,6 +187,31 @@ class Changes(object): if (not self.changes.has_key(key)) or (not self.changes[key]): self.changes[key]='missing' + def __get_file_from_pool(self, filename, entry, session): + cnf = Config() + + poolname = poolify(entry["source"], entry["component"]) + l = get_location(cnf["Dir::Pool"], entry["component"], session=session) + + found, poolfile = check_poolfile(os.path.join(poolname, filename), + entry['size'], + entry["md5sum"], + l.location_id, + session=session) + + if found is None: + Logger.log(["E: Found multiple files for pool (%s) for %s" % (chg_fn, entry["component"])]) + return None + elif found is False and poolfile is not None: + Logger.log(["E: md5sum/size mismatch for %s in pool" % (chg_fn)]) + return None + else: + if poolfile is None: + Logger.log(["E: Could not find %s in pool" % (chg_fn)]) + return None + else: + return poolfile + @session_wrapper def add_known_changes(self, dirpath, in_queue=None, session=None): """add "missing" in fields which we will require for the known_changes table""" @@ -248,27 +273,22 @@ class Changes(object): except IOError: # Can't find the file, try to look it up in the pool - poolname = poolify(entry["source"], entry["component"]) - l = get_location(cnf["Dir::Pool"], entry["component"], session=session) - - found, poolfile = check_poolfile(os.path.join(poolname, chg_fn), - entry['size'], - entry["md5sum"], - l.location_id, - session=session) - - if found is None: - Logger.log(["E: Found multiple files for pool (%s) for %s" % (chg_fn, entry["component"])]) - elif found is False and poolfile is not None: - Logger.log(["E: md5sum/size mismatch for %s in pool" % (chg_fn)]) - else: - if poolfile is None: - Logger.log(["E: Could not find %s in pool" % (chg_fn)]) - else: - chg.poolfiles.append(poolfile) + poolfile = self.__get_file_from_pool(chg_fn, entry, session) + if poolfile: + chg.poolfiles.append(poolfile) chg.files = files + # Add files referenced in .dsc, but not included in .changes + for name, entry in self.dsc_files.items(): + if self.files.has_key(name): + continue + + entry['source'] = self.changes['source'] + poolfile = self.__get_file_from_pool(name, entry, session) + if poolfile: + chg.poolfiles.append(poolfile) + session.commit() chg = session.query(DBChange).filter_by(changesname = self.changes_file).one(); diff --git a/daklib/dbconn.py b/daklib/dbconn.py index fe04ebc3..ae5a9e0d 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -703,6 +703,7 @@ class BuildQueue(object): try: # Grab files we want to include newer = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all() + newer += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) > starttime).all() # Write file list with newer files (fl_fd, fl_name) = mkstemp() for n in newer: @@ -795,6 +796,7 @@ class BuildQueue(object): # Grab files older than our execution time older = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueueFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all() + older += session.query(BuildQueuePolicyFile).filter_by(build_queue_id = self.queue_id).filter(BuildQueuePolicyFile.lastused + timedelta(seconds=self.stay_of_execution) <= starttime).all() for o in older: killdb = False @@ -822,9 +824,7 @@ class BuildQueue(object): if f.startswith('Packages') or f.startswith('Source') or f.startswith('Release') or f.startswith('advisory'): continue - try: - r = session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id).filter_by(filename = f).one() - except NoResultFound: + if not self.contains_filename(f): fp = os.path.join(self.path, f) if dryrun: Logger.log(["I: Would remove unused link %s" % fp]) @@ -835,6 +835,18 @@ class BuildQueue(object): except OSError: Logger.log(["E: Failed to unlink unreferenced file %s" % r.fullpath]) + def contains_filename(self, filename): + """ + @rtype Boolean + @returns True if filename is supposed to be in the queue; False otherwise + """ + session = DBConn().session().object_session(self) + if session.query(BuildQueueFile).filter_by(build_queue_id = self.queue_id, filename = filename).count() > 0: + return True + elif session.query(BuildQueuePolicyFile).filter_by(build_queue = self, filename = filename).count() > 0: + return True + return False + def add_file_from_pool(self, poolfile): """Copies a file into the pool. Assumes that the PoolFile object is attached to the same SQLAlchemy session as the Queue object is. @@ -879,6 +891,61 @@ class BuildQueue(object): return qf + def add_changes_from_policy_queue(self, policyqueue, changes): + """ + Copies a changes from a policy queue together with its poolfiles. + + @type policyqueue: PolicyQueue + @param policyqueue: policy queue to copy the changes from + + @type changes: DBChange + @param changes: changes to copy to this build queue + """ + for policyqueuefile in changes.files: + self.add_file_from_policy_queue(policyqueue, policyqueuefile) + for poolfile in changes.poolfiles: + self.add_file_from_pool(poolfile) + + def add_file_from_policy_queue(self, policyqueue, policyqueuefile): + """ + Copies a file from a policy queue. + Assumes that the policyqueuefile is attached to the same SQLAlchemy + session as the Queue object is. The caller is responsible for + committing after calling this function. + + @type policyqueue: PolicyQueue + @param policyqueue: policy queue to copy the file from + + @type policyqueuefile: ChangePendingFile + @param policyqueuefile: file to be added to the build queue + """ + session = DBConn().session().object_session(policyqueuefile) + + # Is the file already there? + try: + f = session.query(BuildQueuePolicyFile).filter_by(build_queue=self, file=policyqueuefile).one() + f.lastused = datetime.now() + return f + except NoResultFound: + pass # continue below + + # We have to add the file. + f = BuildQueuePolicyFile() + f.build_queue = self + f.file = policyqueuefile + f.filename = policyqueuefile.filename + + source = os.path.join(policyqueue.path, policyqueuefile.filename) + target = f.fullpath + try: + # Always copy files from policy queues as they might move around. + import utils + utils.copy(source, target) + except OSError: + return None + + session.add(f) + return f __all__.append('BuildQueue') @@ -911,6 +978,10 @@ __all__.append('get_build_queue') ################################################################################ class BuildQueueFile(object): + """ + BuildQueueFile represents a file in a build queue coming from a pool. + """ + def __init__(self, *args, **kwargs): pass @@ -926,6 +997,27 @@ __all__.append('BuildQueueFile') ################################################################################ +class BuildQueuePolicyFile(object): + """ + BuildQueuePolicyFile represents a file in a build queue that comes from a + policy queue (and not a pool). + """ + + def __init__(self, *args, **kwargs): + pass + + #@property + #def filename(self): + # return self.file.filename + + @property + def fullpath(self): + return os.path.join(self.build_queue.path, self.filename) + +__all__.append('BuildQueuePolicyFile') + +################################################################################ + class ChangePendingBinary(object): def __init__(self, *args, **kwargs): pass @@ -3080,6 +3172,7 @@ class DBConn(object): 'binary_acl_map', 'build_queue', 'build_queue_files', + 'build_queue_policy_files', 'changelogs_text', 'changes', 'component', @@ -3174,6 +3267,11 @@ class DBConn(object): properties = dict(buildqueue = relation(BuildQueue, backref='queuefiles'), poolfile = relation(PoolFile, backref='buildqueueinstances'))) + mapper(BuildQueuePolicyFile, self.tbl_build_queue_policy_files, + properties = dict( + build_queue = relation(BuildQueue, backref='policy_queue_files'), + file = relation(ChangePendingFile, lazy='joined'))) + mapper(DBBinary, self.tbl_binaries, properties = dict(binary_id = self.tbl_binaries.c.id, package = self.tbl_binaries.c.package, diff --git a/daklib/queue_install.py b/daklib/queue_install.py index b1c2f55e..8878e55c 100755 --- a/daklib/queue_install.py +++ b/daklib/queue_install.py @@ -64,6 +64,14 @@ def package_to_queue(u, summary, short_summary, queue, chg, session, announce=No u.move_to_queue(queue) chg.in_queue_id = queue.policy_queue_id session.add(chg) + + # send to build queues + if queue.send_to_build_queues: + for suite_name in u.pkg.changes["distribution"].keys(): + suite = get_suite(suite_name, session) + for q in suite.copy_queues: + q.add_changes_from_policy_queue(queue, chg) + session.commit() # Check for override disparities @@ -126,11 +134,6 @@ def do_unembargo(u, summary, short_summary, chg, session=None): package_to_queue(u, summary, short_summary, polq, chg, session, announce=None) - for suite_name in u.pkg.changes["distribution"].keys(): - suite = get_suite(suite_name, session) - for q in suite.copy_queues: - for f in u.pkg.files.keys(): - copyfile(os.path.join(polq.path, f), os.path.join(q.path, f)) # ################################################################################# # @@ -152,11 +155,6 @@ def do_embargo(u, summary, short_summary, chg, session=None): package_to_queue(u, summary, short_summary, polq, chg, session, announce=None) - for suite_name in u.pkg.changes["distribution"].keys(): - suite = get_suite(suite_name, session) - for q in suite.copy_queues: - for f in u.pkg.files.keys(): - copyfile(os.path.join(polq.path, f), os.path.join(q.path, f)) ################################################################################