From 87bf163d12ec8328d87e1b2a2ca2239221a73bd6 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Sun, 26 Apr 2009 13:15:37 +0200 Subject: [PATCH] NEW modified process-new and all related stuff so NEW can now also be processed during (most of) dinstall. The lock where NEW is forbidden is then down to some 15 til 20 minutes, instead of 2 hours. For that process-new moves files into a "staging" directory, from which cron.unchecked will move them into the accepted queue. As c.u only runs when it is safe to do so this is fine. The small part where we still are forbidden to do NEW is between the point where we run process-accepted and right before we start generating Packages files. As that is exactly the area daily.lock exists for, process-new now locks on that and simply refuses to do work then. Signed-off-by: Joerg Jaspert --- config/debian/cron.unchecked | 19 ++++++++ config/debian/dak.conf | 3 +- config/debian/vars | 1 + dak/process_new.py | 88 +++++++++++++++++++----------------- daklib/dak_exceptions.py | 3 +- daklib/queue.py | 23 ++++++++-- 6 files changed, 89 insertions(+), 48 deletions(-) diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked index fc146dad..0006f80e 100755 --- a/config/debian/cron.unchecked +++ b/config/debian/cron.unchecked @@ -23,6 +23,25 @@ cleanup() { fi } +# in case there is anything NEW processed, move it over into the accepted queue +cd $newstage +for file in $(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs); do + sed '1,/Files:/d' "${file}" | sed '/BEGIN PGP SIGNATURE/,$d' \ + | while read notused1 notused2 notused3 notused4 NAME; do + if [ -z "${NAME}" ]; then + # Sometimes there is a newline after the Files:, ignore it + continue + fi + if [ -f "${NAME}" ]; then + mv --target-directory="${accepted}" "${NAME}" + else + log_error "Error, couldn't find file ${NAME} to move to ${accepted}" + exit 2 + fi + done + mv --target-directory="${accepted}" "${file}" "${file%%.changes}.dak" +done + # only run one cron.unchecked if lockfile -r3 $LOCKFILE; then trap cleanup 0 diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 756a37c3..baf140e1 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -173,7 +173,7 @@ Clean-Suites Process-New { - AcceptedLockFile "/srv/ftp.debian.org/lock/unchecked.lock"; + DinstallLockFile "/srv/ftp.debian.org/lock/daily.lock"; LockDir "/srv/ftp.debian.org/lock/new/"; }; @@ -563,6 +563,7 @@ Dir New "/srv/ftp.debian.org/queue/new/"; Reject "/srv/ftp.debian.org/queue/reject/"; Unchecked "/srv/ftp.debian.org/queue/unchecked/"; + Newstage "/srv/ftp.debian.org/queue/newstage/"; BTSVersionTrack "/srv/ftp.debian.org/queue/bts_version_track/"; }; }; diff --git a/config/debian/vars b/config/debian/vars index 09a047b2..35e9f0e5 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -23,6 +23,7 @@ unchecked=$queuedir/unchecked/ accepted=$queuedir/accepted/ mirrordir=$base/mirror/ incoming=$base/incoming +newstage=$queuedir/newstage/ ftpgroup=debadmin diff --git a/dak/process_new.py b/dak/process_new.py index 52e1f4e9..d631b53a 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -59,7 +59,7 @@ from daklib import logging from daklib import queue from daklib import utils from daklib.regexes import re_no_epoch, re_default_answer, re_isanum -from daklib.dak_exceptions import CantOpenError, AlreadyLockedError +from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError # Globals Cnf = None #: Configuration, apt_pkg.Configuration @@ -687,7 +687,12 @@ def do_new(): answer = answer[:1].upper() if answer == 'A' and not Options["Trainee"]: - done = add_overrides (new) + try: + check_daily_lock() + done = add_overrides (new) + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'C': check_pkg() elif answer == 'E' and not Options["Trainee"]: @@ -821,22 +826,18 @@ def do_byhand(): ################################################################################ -def get_accept_lock(): - retry = 0 - while retry < 10: - try: - os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) - retry = 10 - except OSError, e: - if e.errno == errno.EACCES or e.errno == errno.EEXIST: - retry += 1 - if (retry >= 10): - utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.") - else: - print("Unable to get accepted lock (try %d of 10)" % retry) - time.sleep(60) - else: - raise +def check_daily_lock(): + """ + Raises CantGetLockError if the dinstall daily.lock exists. + """ + + try: + os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + except OSError, e: + if e.errno == errno.EEXIST or e.errno == errno.EACCES: + raise CantGetLockError + + os.unlink(Cnf["Process-New::DinstallLockFile"]) @contextlib.contextmanager @@ -898,7 +899,7 @@ def _accept(): if Options["No-Action"]: return (summary, short_summary) = Upload.build_summaries() - Upload.accept(summary, short_summary) + Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"]) os.unlink(Upload.pkg.changes_file[:-8]+".dak") def do_accept_stableupdate(suite, q): @@ -944,30 +945,25 @@ def do_accept_stableupdate(suite, q): def do_accept(): print "ACCEPT" if not Options["No-Action"]: - get_accept_lock() (summary, short_summary) = Upload.build_summaries() - try: - if Cnf.FindB("Dinstall::SecurityQueueHandling"): - Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) - move_to_dir(Cnf["Dir::Queue::Embargoed"]) - Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) - # Check for override disparities - Upload.Subst["__SUMMARY__"] = summary - else: - # Stable updates need to be copied to proposed-updates holding - # area instead of accepted. Sourceful uploads need to go - # to it directly, binaries only if the source has not yet been - # accepted into p-u. - for suite, q in [("proposed-updates", "ProposedUpdates"), - ("oldstable-proposed-updates", "OldProposedUpdates")]: - if not Upload.pkg.changes["distribution"].has_key(suite): - continue - return do_accept_stableupdate(suite, q) - # Just a normal upload, accept it... - _accept() - finally: - if not Options["No-Action"]: - os.unlink(Cnf["Process-New::AcceptedLockFile"]) + if Cnf.FindB("Dinstall::SecurityQueueHandling"): + Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) + move_to_dir(Cnf["Dir::Queue::Embargoed"]) + Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) + # Check for override disparities + Upload.Subst["__SUMMARY__"] = summary + else: + # Stable updates need to be copied to proposed-updates holding + # area instead of accepted. Sourceful uploads need to go + # to it directly, binaries only if the source has not yet been + # accepted into p-u. + for suite, q in [("proposed-updates", "ProposedUpdates"), + ("oldstable-proposed-updates", "OldProposedUpdates")]: + if not Upload.pkg.changes["distribution"].has_key(suite): + continue + return do_accept_stableupdate(suite, q) + # Just a normal upload, accept it... + _accept() def check_status(files): new = byhand = 0 @@ -985,6 +981,13 @@ def do_pkg(changes_file): Upload.update_subst() files = Upload.pkg.files + try: + check_daily_lock() + except CantGetLockError: + print "Hello? Operator! Give me the number for 911!" + print "Dinstall in the locked area, cant process packages, come back later" + sys.exit(1) + try: with lock_package(Upload.pkg.changes["source"]): if not recheck(): @@ -1100,6 +1103,7 @@ def main(): if not changes_file: continue print "\n" + changes_file + do_pkg (changes_file) end() diff --git a/daklib/dak_exceptions.py b/daklib/dak_exceptions.py index 21fce9be..ef24fd75 100755 --- a/daklib/dak_exceptions.py +++ b/daklib/dak_exceptions.py @@ -61,7 +61,8 @@ dakerrors = { "MissingContents": """Exception raised - we could not determine contents for this deb""", "DBUpdateError": """Exception raised - could not update the database""", "ChangesUnicodeError": """Exception raised - changes file not properly utf-8 encoded""", - "AlreadyLockedError": """Exception raised - package already locked by someone else""" + "AlreadyLockedError": """Exception raised - package already locked by someone else""", + "CantGetLockError": """Exception raised - lockfile already in use""" } #: All dak exceptions def construct_dak_exception(name, description): diff --git a/daklib/queue.py b/daklib/queue.py index 35754c8d..ceb4a797 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -552,7 +552,7 @@ distribution.""" ########################################################################### - def accept (self, summary, short_summary): + def accept (self, summary, short_summary, targetdir=None): """ Accept an upload. @@ -577,16 +577,19 @@ distribution.""" changes_file = self.pkg.changes_file dsc = self.pkg.dsc + if targetdir is None: + targetdir = Cnf["Dir::Queue::Accepted"] + print "Accepting." self.Logger.log(["Accepting changes",changes_file]) - self.dump_vars(Cnf["Dir::Queue::Accepted"]) + self.dump_vars(targetdir) # Move all the files into the accepted directory - utils.move(changes_file, Cnf["Dir::Queue::Accepted"]) + utils.move(changes_file, targetdir) file_keys = files.keys() for file_entry in file_keys: - utils.move(file_entry, Cnf["Dir::Queue::Accepted"]) + utils.move(file_entry, targetdir) self.accept_bytes += float(files[file_entry]["size"]) self.accept_count += 1 @@ -635,6 +638,18 @@ distribution.""" os.rename(temp_filename, filename) os.chmod(filename, 0644) + # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir! + # we do call queue_build too + # well yes, we'd have had to if we were inserting into accepted + # now. thats database only. + # urgh, that's going to get messy + # so i make the p-n call to it *also* using accepted/ + # but then the packages will be in the queue_build table without the files being there + # as the buildd queue is only regenerated whenever unchecked runs + # ah, good point + # so it will work out, as unchecked move it over + # that's all completely sick + # yes self.queue_build("accepted", Cnf["Dir::Queue::Accepted"]) ########################################################################### -- 2.39.2