From: Anthony Towns Date: Fri, 21 Mar 2008 13:59:45 +0000 (+0000) Subject: merge with mainline X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=1ec6fce21325f5b3be284dd63981e6574e22d223;hp=01f8d59474ed7614812ad0bed72ce9f24d6df72a;p=dak.git merge with mainline --- diff --git a/ChangeLog b/ChangeLog index 59daedac..d269ff65 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,58 @@ * config/debian/extensions.py: Add infrastructure for replacing functions in dak modules; add upload blocking for dpkg. +2008-03-12 Joerg Jaspert + + * dak/edit_transitions.py: Done a number of cleanups to make code + working. Also changed the way prompting/answering goes, to not + have to import daklib/queue. + (edit_transitions): When done with a successful edit - also print + a final overview about defined transitions + +2008-03-11 Joerg Jaspert + + * dak/process_unchecked.py: Import syck module directly, not "from + syck import *" + (check_transition): Do the check for sourceful upload in here + Also adjust the syck loading commands, rename new_vers to + expected, curvers to current, to make it more clear what they mean. + + * daklib/database.py (get_suite_version): Renamed from + get_testing_version. Also changed the cache variables name + + * The above changes are based on modifications from Anthony. + + * dak/dak.py (init): Renamed check -> edit transitions + + * dak/edit_transitions.py: Renamed from check_transitions.py + (main): Also rename new_vers/curvers to expected/current + Basically a nice rewrite, so it now does checks and edit, + depending on how you call it. Check also removes old transitions, + if user wants it. + +2008-03-02 Joerg Jaspert + + * debian/control (Suggests): Add python-syck to Depends: + + * dak/dak.py (init): Tell it about check_transitions + + * dak/check_transitions.py (usage): Added, checks the transitions + file (if any) + + * daklib/database.py (get_testing_version): Added. Returns the + version for the source in testing, if any + + * dak/process_unchecked.py (check_transition): Added. Checks if a + release team member defined a transition, and rejects based on + that data. + (process_it): Use it. + (check_transition): Warn on broken transitions file and return, + not doing anything. + (check_transition): Moved out of here, into daklib/queue + (process_it): Call check_transitions only if + changes[architecture] has source included. + (check_transition): Now call the database.get_testing_version + 2008-02-06 Joerg Jaspert * daklib/utils.py (check_signature): Make variable key available, diff --git a/config/debian/dak.conf b/config/debian/dak.conf index a62a5109..e9f52e2f 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -34,6 +34,7 @@ Dinstall Reject { NoSourceOnly "true"; + ReleaseTransitions "/srv/ftp.debian.org/testing/hints/transitions.yaml"; }; }; diff --git a/config/debian/extensions.py b/config/debian/extensions.py index 9d6bef1c..4340ca3b 100644 --- a/config/debian/extensions.py +++ b/config/debian/extensions.py @@ -1,8 +1,81 @@ -import sys, os +import sys, os, textwrap + +import apt_pkg +import daklib.utils, daklib.database +import syck import daklib.extensions from daklib.extensions import replace_dak_function +def check_transition(): + changes = dak_module.changes + reject = dak_module.reject + Cnf = dak_module.Cnf + + sourcepkg = changes["source"] + + # No sourceful upload -> no need to do anything else, direct return + if "source" not in changes["architecture"]: + return + + # Also only check if there is a file defined (and existant) with + # checks. + transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "") + if transpath == "" or not os.path.exists(transpath): + return + + # Parse the yaml file + sourcefile = file(transpath, 'r') + sourcecontent = sourcefile.read() + try: + transitions = syck.load(sourcecontent) + except syck.error, msg: + # This shouldn't happen, there is a wrapper to edit the file which + # checks it, but we prefer to be safe than ending up rejecting + # everything. + daklib.utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg)) + return + + # Now look through all defined transitions + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + if current is not None: + compare = apt_pkg.VersionCompare(current, expected) + + if current is None or compare < 0: + # This is still valid, the current version in testing is older than + # the new version we wait for, or there is none in testing yet + + # Check if the source we look at is affected by this. + if sourcepkg in t['packages']: + # The source is affected, lets reject it. + + rejectmsg = "%s: part of the %s transition.\n\n" % ( + sourcepkg, trans) + + if current is not None: + currentlymsg = "at version %s" % (current) + else: + currentlymsg = "not present in testing" + + rejectmsg += "Transition description: %s\n\n" % (t["reason"]) + + rejectmsg += "\n".join(textwrap.wrap("""Your package +is part of a testing transition designed to get %s migrated (it is +currently %s, we need version %s). This transition is managed by the +Release Team, and %s is the Release-Team member responsible for it. +Please mail debian-release@lists.debian.org or contact %s directly if you +need further assistance.""" + % (source, currentlymsg, expected,t["rm"], t["rm"]))) + + reject(rejectmsg + "\n") + return + @replace_dak_function("process-unchecked", "check_signed_by_key") def check_signed_by_key(oldfn): changes = dak_module.changes @@ -21,4 +94,4 @@ def check_signed_by_key(oldfn): oldfn() - + check_transition() diff --git a/dak/dak.py b/dak/dak.py index 2e4bd820..43da4dcb 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -91,7 +91,9 @@ def init(): "Clean cruft from incoming"), ("clean-proposed-updates", "Remove obsolete .changes from proposed-updates"), - + + ("edit-transitions", + "Edit the release transition file"), ("check-overrides", "Override cruft checks"), ("check-proposed-updates", diff --git a/dak/edit_transitions.py b/dak/edit_transitions.py new file mode 100755 index 00000000..15cc8dc4 --- /dev/null +++ b/dak/edit_transitions.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python + +# Edit and then check the release managers transition file for correctness +# and outdated transitions +# Copyright (C) 2008 Joerg Jaspert + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# if klecker.d.o died, I swear to god, I'm going to migrate to gentoo. + +################################################################################ + +import os, pg, sys, time +import apt_pkg +import daklib.database +import daklib.utils +import syck + +# Globals +Cnf = None +Options = None +projectB = None + +################################################################################ + +def init(): + global Cnf, Options, projectB + + apt_pkg.init() + + Cnf = daklib.utils.get_conf() + + Arguments = [('h',"help","Edit-Transitions::Options::Help"), + ('e',"edit","Edit-Transitions::Options::Edit"), + ('c',"check","Edit-Transitions::Options::check"), + ('n',"no-action","Edit-Transitions::Options::No-Action")] + + for i in ["help", "no-action", "edit", "check"]: + if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)): + Cnf["Edit-Transitions::Options::%s" % (i)] = "" + + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) + + Options = Cnf.SubTree("Edit-Transitions::Options") + + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + daklib.database.init(Cnf, projectB) + + if Options["help"]: + usage() + +################################################################################ + +def usage (exit_code=0): + print """Usage: edit_transitions [OPTION]... + Check the release managers transition file for correctness and outdated transitions + -h, --help show this help and exit. + -e, --edit edit the transitions file + -c, --check check the transitions file, remove outdated entries + -n, --no-action don't do anything + + Called without an option this tool will check the transition file for outdated + transitions and remove them.""" + sys.exit(exit_code) + +################################################################################ + +def lock_file(lockfile): + retry = 0 + while retry < 10: + try: + lock_fd = os.open(lockfile, os.O_RDONLY | os.O_CREAT | os.O_EXCL) + retry = 10 + except OSError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': + retry += 1 + if (retry >= 10): + daklib.utils.fubar("Couldn't obtain lock for %s." % (lockfile) ) + else: + print("Unable to get lock for %s (try %d of 10)" % (lockfile, retry) ) + time.sleep(60) + else: + raise + + +################################################################################ + +def edit_transitions(): + trans_file = Cnf["Dinstall::Reject::ReleaseTransitions"] + + tempfile = "./%s.transition.tmp" % (os.getpid() ) + + lockfile="%s.lock" % (tempfile) + lock_file(lockfile) + + daklib.utils.copy(trans_file, tempfile) + + editor = os.environ.get("EDITOR", "vi") + + while True: + result = os.system("%s %s" % (editor, tempfile)) + if result != 0: + os.unlink(tempfile) + os.unlink(lockfile) + daklib.utils.fubar("%s invocation failed for %s, not removing tempfile." % (editor, tempfile)) + + # Now try to load the new file + test = load_transitions(tempfile) + + if test == None: + # Edit is broken + answer = "XXX" + prompt = "Broken edit: [E]dit again, Drop changes?" + + while prompt.find(answer) == -1: + answer = daklib.utils.our_raw_input(prompt) + if answer == "": + answer = "E" + answer = answer[:1].upper() + + if answer == 'E': + continue + elif answer == 'D': + os.unlink(tempfile) + os.unlink(lockfile) + print "OK, discarding changes" + sys.exit(0) + else: + # No problems in loading the new file, jump out of the while loop + break + + # We seem to be done and also have a working file. Copy over. + daklib.utils.copy(tempfile, trans_file, True) + os.unlink(tempfile) + os.unlink(lockfile) + + # Before we finish print out transition info again + print "\n\n------------------------------------------------------------------------" + print "Edit done, file saved, currently defined transitions:\n" + transitions = load_transitions(Cnf["Dinstall::Reject::ReleaseTransitions"]) + transition_info(transitions) + +################################################################################ + +def load_transitions(trans_file): + # Parse the yaml file + sourcefile = file(trans_file, 'r') + sourcecontent = sourcefile.read() + try: + trans = syck.load(sourcecontent) + except syck.error, msg: + # Someone fucked it up + print "ERROR: %s" % (msg) + return None + return trans + +################################################################################ + +def print_info(trans, source, expected, rm, reason, packages): + print """ +Looking at transition: %s + Source: %s + New Version: %s + Responsible: %s + Description: %s + Blocked Packages (total: %d): %s +""" % (trans, source, expected, rm, reason, len(packages), ", ".join(packages)) + return + +################################################################################ + +def transition_info(transitions): + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + + print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + + if current == None: + # No package in testing + print "Transition source %s not in testing, transition still ongoing." % (source) + else: + compare = apt_pkg.VersionCompare(current, expected) + print "Apt compare says: %s" % (compare) + if compare < 0: + # This is still valid, the current version in database is older than + # the new version we wait for + print "This transition is still ongoing, we currently have version %s" % (current) + else: + print "This transition is over, the target package reached testing, should be removed" + print "%s wanted version: %s, has %s" % (source, expected, current) + print "-------------------------------------------------------------------------" + +################################################################################ + +def check_transitions(transitions): + to_dump = 0 + to_remove = [] + # Now look through all defined transitions + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + + print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + + if current == None: + # No package in testing + print "Transition source %s not in testing, transition still ongoing." % (source) + else: + compare = apt_pkg.VersionCompare(current, expected) + print "Apt compare says: %s" % (compare) + if compare < 0: + # This is still valid, the current version in database is older than + # the new version we wait for + print "This transition is still ongoing, we currently have version %s" % (current) + else: + print "REMOVE: This transition is over, the target package reached testing. REMOVE" + print "%s wanted version: %s, has %s" % (source, expected, current) + to_remove.append(trans) + to_dump = 1 + print "-------------------------------------------------------------------------" + + if to_dump: + prompt = "Removing: " + for remove in to_remove: + prompt += remove + prompt += "," + + prompt += " Commit Changes? (y/N)" + answer = "" + + if Options["no-action"]: + answer="n" + else: + answer = daklib.utils.our_raw_input(prompt).lower() + + if answer == "": + answer = "n" + + if answer == 'n': + print "Not committing changes" + sys.exit(0) + elif answer == 'y': + print "Committing" + for remove in to_remove: + del transitions[remove] + destfile = file(Cnf["Dinstall::Reject::ReleaseTransitions"], 'w') + syck.dump(transitions, destfile) + print "Done" + else: + print "WTF are you typing?" + sys.exit(0) + + +################################################################################ + +def main(): + global Cnf + + init() + + # Only check if there is a file defined (and existant) with checks. It's a little bit + # specific to Debian, not much use for others, so return early there. + if not Cnf.has_key("Dinstall::Reject::ReleaseTransitions") or not os.path.exists("%s" % (Cnf["Dinstall::Reject::ReleaseTransitions"])): + daklib.utils.warn("Dinstall::Reject::ReleaseTransitions not defined or file %s not existant." % + (Cnf["Dinstall::Reject::ReleaseTransitions"])) + sys.exit(1) + + # Parse the yaml file + transitions = load_transitions(Cnf["Dinstall::Reject::ReleaseTransitions"]) + if transitions == None: + # Something very broken with the transitions, exit + daklib.utils.warn("Not doing any work, someone fucked up the transitions file outside our control") + sys.exit(2) + + if Options["edit"]: + # Output information about the currently defined transitions. + print "Currently defined transitions:" + transition_info(transitions) + daklib.utils.our_raw_input("Press enter to continue...") + + # Lets edit the transitions file + edit_transitions() + elif Options["check"]: + # Check and remove outdated transitions + check_transitions(transitions) + else: + # Output information about the currently defined transitions. + transition_info(transitions) + + # Nothing requested, doing nothing besides the above display of the transitions + sys.exit(0) + + +################################################################################ + +if __name__ == '__main__': + main() diff --git a/daklib/database.py b/daklib/database.py old mode 100644 new mode 100755 index a40696e2..5c362604 --- a/daklib/database.py +++ b/daklib/database.py @@ -41,6 +41,7 @@ maintainer_cache = {} fingerprint_id_cache = {} queue_id_cache = {} uid_id_cache = {} +suite_version_cache = {} ################################################################################ @@ -223,6 +224,29 @@ def get_source_id (source, version): return source_id +def get_suite_version(source, suite): + global suite_version_cache + cache_key = "%s_%s" % (source, suite) + + if suite_version_cache.has_key(cache_key): + return suite_version_cache[cache_key] + + q = projectB.query(""" + SELECT s.version FROM source s, suite su, src_associations sa + WHERE sa.source=s.id + AND sa.suite=su.id + AND su.suite_name='%s' + AND s.source='%s'""" + % (suite, source)) + + if not q.getresult(): + return None + + version = q.getresult()[0][0] + suite_version_cache[cache_key] = version + + return version + ################################################################################ def get_or_set_maintainer_id (maintainer): diff --git a/debian/control b/debian/control index 97c1fbe1..2d6678b9 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Standards-Version: 3.5.6.0 Package: dak Architecture: any -Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev +Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev, python-syck (>= 0.61.2-1) Suggests: lintian, linda, less, binutils-multiarch, symlinks, postgresql (>= 7.1.0), dsync Description: Debian's archive maintenance scripts This is a collection of archive maintenance scripts used by the diff --git a/docs/transitions.txt b/docs/transitions.txt new file mode 100644 index 00000000..5b52bb31 --- /dev/null +++ b/docs/transitions.txt @@ -0,0 +1,275 @@ +Contents: + +1. Little "Howto Use it" +2. Explanation of how it works + + +1. Little "Howto Use it" +------------------------ + +The input file is in YAML format. Do bnot bother with comments, they +will be removed. + +The format: Dont use tabs for indentation, use spaces. + +Strings should be within "", but normally work without. +Exception: Version-numbers with an epoch really do want to be in +"". YES, THEY WANT TO (or they get interpreted in a way you dont expect +it). + +Keys (The order of them does not matter, only the indentation): + +short_tag: A short tag for the transition, like apt_update + reason: One-line reason what is intended with it + source: Source package that needs to transition + new: New version of the target package + rm: Name of the Release Team member responsible for this transition + packages: Array of package names that are affected by this transition + + +The following example wants to +a.) update apt to version 0.7.12, the responsible Release Team member +is Andreas Barth, and it affects some apt related packages and +b.) wants to do something similar for lintian. + +apt_update: + packages: + - apt + - synaptic + - cron-apt + - debtags + - feta + - apticron + - aptitude + reason: "Apt needs to transition to testing to get foo and bar done" + source: apt + new: 0.7.12 + rm: Andreas Barth +lintian_breakage: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck + + +######################################################################## +######################################################################## + + +2. Explanation of how it works +------------------------------ + +Assume the following transition is defined: + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck + +Also assume the lintian situation on this archive looks like this: + lintian | 1.23.28~bpo.1 | sarge-backports | source, all + lintian | 1.23.45~bpo40+1 | etch-backports | source, all + +------------------------------------------------------------------------ + +Now, I try to upload a (NEW, but that makes no difference) version of +python-syck: + +$ dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +REJECT +Rejected: python-syck: part of the lintian_funtest transition. + +Your package is part of a testing transition designed to get lintian migrated +(it currently is at version 1.23.28~bpo.1, we need version 1.23.45~bpo40+1) + +Transition description: Testing a new feature + +This transition is managed by the Release Team, and Ganneff +is the Release-Team member responsible for it. +Please contact Ganneff or debian-release@lists.debian.org if you +need further assistance. + +------------------------------------------------------------------------ + +Lets change the definition of the transition, assume it is now: + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.22.28~bpo.1 + rm: Ganneff + packages: + - lintian + - python-syck + +Which checks for a version older than the version actually available. Result: + +dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +NEW for etch-backports +(new) python-syck_0.61.2-1~bpo40+1.diff.gz extra python +(new) python-syck_0.61.2-1~bpo40+1.dsc extra python +(new) python-syck_0.61.2-1~bpo40+1_i386.deb extra python +PySyck python bindings to the Syck YAML parser kit + Syck is a simple YAML parser kit. + . +[...] the whole stuff about a new package. + +------------------------------------------------------------------------ + +For completness, change the transition to (exact existing version): +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.28~bpo.1 + rm: Ganneff + packages: + - lintian + +and the result is: + +dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +NEW for etch-backports +[... we know this ...] + +------------------------------------------------------------------------ + +The second part is the check_transitions script. +For that we take the following transitions as example: + +apt_update: + reason: "Apt needs to transition to testing to get foo and bar done" + source: apt + new: 0.2.12-1+b1.3 + rm: Andreas Barth + packages: + - apt + - synaptic + - cron-apt + - debtags + - feta + - apticron + - aptitude +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck +bar_breaks_it: + reason: We dont want bar to break it + source: bar + new: "9:99" + rm: Ganneff + packages: + - kdelibs + - qt4-x11 + - libqt-perl + +Running check-transitions ends up with the following output: + +Looking at transition: lintian_funtest + Source: lintian + New Version: 1.23.45~bpo40+1 + Responsible: Ganneff + Description: Testing a new feature + Blocked Packages (total: 2): lintian, python-syck + +Apt compare says: -2 +This transition is still ongoing, we currently have version 1.23.28~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: apt_update + Source: apt + New Version: 0.2.12-1+b1.3 + Responsible: Andreas Barth + Description: Apt needs to transition to testing to get foo and bar done + Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude + +Apt compare says: 4 +This transition is over, the target package reached testing, removing +apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: bar_breaks_it + Source: bar + New Version: 9:99 + Responsible: Ganneff + Description: We dont want bar to break it + Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl + +Transition source bar not in testing, transition still ongoing. +------------------------------------------------------------------------- +I: I would remove the apt_update transition + + +Changing our transition definitions for lintian (keeping the rest as +above) to + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.22.28~bpo.1 + rm: Ganneff + packages: + - lintian + - python-syck + +now we get + +Looking at transition: lintian_funtest + Source: lintian + New Version: 1.22.28~bpo.1 + Responsible: Ganneff + Description: Testing a new feature + Blocked Packages (total: 2): lintian, python-syck + +Apt compare says: 1 +This transition is over, the target package reached testing, removing +lintian wanted version: 1.22.28~bpo.1, has 1.23.28~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: apt_update + Source: apt + New Version: 0.2.12-1+b1.3 + Responsible: Andreas Barth + Description: Apt needs to transition to testing to get foo and bar done + Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude + +Apt compare says: 4 +This transition is over, the target package reached testing, removing +apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: bar_breaks_it + Source: bar + New Version: 9:99 + Responsible: Ganneff + Description: We dont want bar to break it + Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl + +Transition source bar not in testing, transition still ongoing. +------------------------------------------------------------------------- +I: I would remove the lintian_funtest transition +I: I would remove the apt_update transition + + +Not using the -n switch would turn the I: in actual removals :) +The check-transition command is meant for the release team to always run +it when they change a transition definition. It checks if the yaml is +valid and can be loaded (but if not the archive simply does no reject) +and also shows a nice overview.