From: Joerg Jaspert Date: Wed, 23 Apr 2008 18:44:09 +0000 (+0200) Subject: Merge with upstream X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=330833cdf958e43962e208f8021e0e44f1fc100c;hp=1ee623994132a49a0251c23083dca750803b817f;p=dak.git Merge with upstream --- diff --git a/ChangeLog b/ChangeLog index f86099a6..8b132e72 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,148 @@ +2008-04-22 Joerg Jaspert + + * setup/init_pool.sql: added a function/aggregate for the release + team to base some script on it. + + * config/debian/cron.daily: push katie@merkel to immediately start + the sync of projectb there. + +2008-04-21 Joerg Jaspert + + * scripts/debian/expire_dumps: New script, expires old database + dumps, using a scheme to keep more of the recent dumps. + + * config/debian/cron.daily: Use the new script. Also - compress + all files older than 7 days, instead of 30. + + * dak/process_accepted.py (install): Do not break if a + source/maintainer combination is already in src_uploaders, "just" + warn us. + +2008-04-20 Thomas Viehmann + + * daklib/utils.py (build_file_list): Deal with "Format 3 style" + Format lines (ie. those having extra text appended). + +2008-04-18 Joerg Jaspert + + * config/debian/dak.conf: Add mapping stable-proposed-updates + -> proposed-updates. + + * dak/transitions.py (load_transitions): Additionally check for + invalid package list indentation + +2008-04-17 Joerg Jaspert + + * config/debian/dak.conf: Add TempPath statement for the Release + Transitions script + + * dak/transitions.py (temp_transitions_file): Use the TempPath + (write_transitions_from_file): Check if the file we should get our + transitions from is in our TempPath, error out if it isnt + (main): Check for TempPath existance + +2008-04-12 James Troup + + * dak/clean_proposed_updates.py: add support for -s/--suite and + -n/--no-action. + +2008-04-11 Anthony Towns + + * dak/utils.py: build_file_list() extra parameters so it can + build a file list for checksums-foo fields. Don't use float() to + compare formats, because Format: 1.10 should compare greater than + Format: 1.9 (use "1.9".split(".",1) and tuple comparison instead) + + * dak/process_unchecked.py: check_md5sum becomes check_hashes + and check_hash. If changes format is 1.8 or later, also check + checksums-sha1 and checksums-sha256 for both .changes and .dsc, + and reject on presence/absence of un/expected checksums-* fields. + +2008-04-07 Joerg Jaspert + + * daklib/utils.py (build_file_list): Check for dpkg .changes + adjusted to reject newer (and right now broken) 1.8 version, until + dpkg (or debsign) is fixed and doesn't produce invalid .changes anymore + +2008-03-22 Joerg Jaspert + + * dak/transitions.py (load_transitions): Check if all our keys are + defined, if there are only keys defined we want and also the types + of the various keys. + +2008-03-22 Anthony Towns + + * dak/edit_transitions.py: Add --import option. + Add --use-sudo option. Use fcntl locking for writing. + Move writing into a function (write_transitions). + Reinvoke self using sudo and --import if necessary. + Move temporary file creation into a function, use mkstemp. + Rename to "dak transitions". + +2008-03-21 Joerg Jaspert + + * dak/edit_transitions.py (edit_transitions): Use sudo to copy the + edited file back in place + (check_transitions): Use proper locking and also use sudo to copy + the new file in place + +2008-03-21 Anthony Towns + + * config/debian/extensions.py: Add infrastructure for replacing + functions in dak modules; add upload blocking for dpkg. + +2008-03-12 Joerg Jaspert + + * dak/edit_transitions.py: Done a number of cleanups to make code + working. Also changed the way prompting/answering goes, to not + have to import daklib/queue. + (edit_transitions): When done with a successful edit - also print + a final overview about defined transitions + +2008-03-11 Joerg Jaspert + + * dak/process_unchecked.py: Import syck module directly, not "from + syck import *" + (check_transition): Do the check for sourceful upload in here + Also adjust the syck loading commands, rename new_vers to + expected, curvers to current, to make it more clear what they mean. + + * daklib/database.py (get_suite_version): Renamed from + get_testing_version. Also changed the cache variables name + + * The above changes are based on modifications from Anthony. + + * dak/dak.py (init): Renamed check -> edit transitions + + * dak/edit_transitions.py: Renamed from check_transitions.py + (main): Also rename new_vers/curvers to expected/current + Basically a nice rewrite, so it now does checks and edit, + depending on how you call it. Check also removes old transitions, + if user wants it. + +2008-03-02 Joerg Jaspert + + * debian/control (Suggests): Add python-syck to Depends: + + * dak/dak.py (init): Tell it about check_transitions + + * dak/check_transitions.py (usage): Added, checks the transitions + file (if any) + + * daklib/database.py (get_testing_version): Added. Returns the + version for the source in testing, if any + + * dak/process_unchecked.py (check_transition): Added. Checks if a + release team member defined a transition, and rejects based on + that data. + (process_it): Use it. + (check_transition): Warn on broken transitions file and return, + not doing anything. + (check_transition): Moved out of here, into daklib/queue + (process_it): Call check_transitions only if + changes[architecture] has source included. + (check_transition): Now call the database.get_testing_version + 2008-02-09 Christoph Berg * daklib/queue.py (get_type): fubar does not exist in global diff --git a/config/debian/apt.conf b/config/debian/apt.conf index b85e4f03..408e7dce 100644 --- a/config/debian/apt.conf +++ b/config/debian/apt.conf @@ -50,7 +50,7 @@ tree "dists/testing" FileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list"; SourceFileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_source.list"; Sections "main contrib non-free"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source"; BinOverride "override.lenny.$(SECTION)"; ExtraOverride "override.lenny.extra.$(SECTION)"; SrcOverride "override.lenny.$(SECTION).src"; @@ -61,7 +61,7 @@ tree "dists/testing-proposed-updates" FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list"; SourceFileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source"; BinOverride "override.lenny.$(SECTION)"; ExtraOverride "override.lenny.extra.$(SECTION)"; SrcOverride "override.lenny.$(SECTION).src"; @@ -109,7 +109,7 @@ tree "dists/testing/main" { FileList "/srv/ftp.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc"; BinOverride "override.lenny.main.$(SECTION)"; SrcOverride "override.lenny.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; @@ -121,7 +121,7 @@ tree "dists/testing/non-free" { FileList "/srv/ftp.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc"; BinOverride "override.lenny.main.$(SECTION)"; SrcOverride "override.lenny.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; @@ -133,7 +133,7 @@ tree "dists/testing-proposed-updates/main" { FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; - Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc"; + Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc"; BinOverride "override.lenny.main.$(SECTION)"; SrcOverride "override.lenny.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; diff --git a/config/debian/cron.daily b/config/debian/cron.daily index b1af5312..48f74687 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -89,8 +89,7 @@ dak make-suite-file-list TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Update fingerprints -# [JT - disabled, dak import-ldap-fingerprints currently can ask questions] -#dak import-ldap-fingerprints +dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg # Generate override files cd $overridedir @@ -158,7 +157,14 @@ pg_dump projectb > $POSTDUMP TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Vacuum the database -echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" +# (JJ, 20-04-2008) disabled, as we have autovacuum set to on in postgres. +# refer to http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM +# which says "Beginning in PostgreSQL 8.1, there is an optional feature called autovacuum, +# whose purpose is to automate the execution of VACUUM and ANALYZE commands." +# echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" + +echo "Expiring old database dumps..." +(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") ################################################################################ @@ -173,6 +179,9 @@ $scriptsdir/dm-monitor >$webdir/dm-uploaders.html ################################################################################ +# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached +ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 + # Run mirror-split #time dak mirror-split @@ -203,10 +212,10 @@ apt-ftparchive -q clean apt.conf TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -# Compress psql backups older than a month, but no more than 20 of them +# Compress psql backups older than a week, but no more than 20 of them (cd $base/backup/ - find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +30 | + find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | sort | head -n20 | while read dumpname; do echo "Compressing $dumpname" bzip2 -9 "$dumpname" diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 6a989215..d5a7df21 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -34,9 +34,15 @@ Dinstall Reject { NoSourceOnly "true"; + ReleaseTransitions "/srv/ftp.debian.org/testing/hints/transitions.yaml"; }; }; +Transitions +{ + TempPath "/srv/ftp.debian.org/tmp/"; +}; + Binary-Upload-Restrictions { Components @@ -221,9 +227,9 @@ Suite sparc; }; Announce "debian-changes@lists.debian.org"; - Version "3.1r6"; + Version "3.1r8"; Origin "Debian"; - Description "Debian 3.1r6 Released 7 April 2007"; + Description "Debian 3.1r8 Released 12 April 2008"; CodeName "sarge"; OverrideCodeName "sarge"; Priority "2"; @@ -407,6 +413,7 @@ Suite alpha; amd64; arm; + armel; hppa; i386; ia64; @@ -444,6 +451,7 @@ Suite alpha; amd64; arm; + armel; hppa; i386; ia64; @@ -628,6 +636,7 @@ SuiteMappings "map oldstable-security oldstable-proposed-updates"; "map stable proposed-updates"; "map stable-security proposed-updates"; + "map stable-proposed-updates proposed-updates"; "map-unreleased oldstable unstable"; "map-unreleased stable unstable"; "map-unreleased proposed-updates unstable"; @@ -639,7 +648,7 @@ SuiteMappings AutomaticByHandPackages { "debian-installer-images" { - Source "xxx-debian-installer"; + Source "debian-installer"; Section "raw-installer"; Extension "tar.gz"; Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-di"; diff --git a/config/debian/extensions.py b/config/debian/extensions.py index 44bd5c72..e17e9af8 100644 --- a/config/debian/extensions.py +++ b/config/debian/extensions.py @@ -1,2 +1,100 @@ -import sys +import sys, os, textwrap +import apt_pkg +import daklib.utils, daklib.database +import syck + +import daklib.extensions +from daklib.extensions import replace_dak_function + +def check_transition(): + changes = dak_module.changes + reject = dak_module.reject + Cnf = dak_module.Cnf + + sourcepkg = changes["source"] + + # No sourceful upload -> no need to do anything else, direct return + # We also work with unstable uploads, not experimental or those going to some + # proposed-updates queue + if "source" not in changes["architecture"] or "unstable" not in changes["distribution"]: + return + + # Also only check if there is a file defined (and existant) with + # checks. + transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "") + if transpath == "" or not os.path.exists(transpath): + return + + # Parse the yaml file + sourcefile = file(transpath, 'r') + sourcecontent = sourcefile.read() + try: + transitions = syck.load(sourcecontent) + except syck.error, msg: + # This shouldn't happen, there is a wrapper to edit the file which + # checks it, but we prefer to be safe than ending up rejecting + # everything. + daklib.utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg)) + return + + # Now look through all defined transitions + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + if current is not None: + compare = apt_pkg.VersionCompare(current, expected) + + if current is None or compare < 0: + # This is still valid, the current version in testing is older than + # the new version we wait for, or there is none in testing yet + + # Check if the source we look at is affected by this. + if sourcepkg in t['packages']: + # The source is affected, lets reject it. + + rejectmsg = "%s: part of the %s transition.\n\n" % ( + sourcepkg, trans) + + if current is not None: + currentlymsg = "at version %s" % (current) + else: + currentlymsg = "not present in testing" + + rejectmsg += "Transition description: %s\n\n" % (t["reason"]) + + rejectmsg += "\n".join(textwrap.wrap("""Your package +is part of a testing transition designed to get %s migrated (it is +currently %s, we need version %s). This transition is managed by the +Release Team, and %s is the Release-Team member responsible for it. +Please mail debian-release@lists.debian.org or contact %s directly if you +need further assistance. You might want to upload to experimental until this +transition is done.""" + % (source, currentlymsg, expected,t["rm"], t["rm"]))) + + reject(rejectmsg + "\n") + return + +@replace_dak_function("process-unchecked", "check_signed_by_key") +def check_signed_by_key(oldfn): + changes = dak_module.changes + reject = dak_module.reject + + if changes["source"] == "dpkg": + fpr = changes["fingerprint"] + (uid, uid_name) = dak_module.lookup_uid_from_fingerprint(fpr) + if fpr == "5906F687BD03ACAD0D8E602EFCF37657" or uid == "iwj": + reject("Upload blocked due to hijack attempt 2008/03/19") + + # NB: 1.15.0, 1.15.2 signed by this key targetted at unstable + # have been made available in the wild, and should remain + # blocked until Debian's dpkg has revved past those version + # numbers + + oldfn() + + check_transition() diff --git a/dak/clean_proposed_updates.py b/dak/clean_proposed_updates.py index a911f899..278dfdf6 100755 --- a/dak/clean_proposed_updates.py +++ b/dak/clean_proposed_updates.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # Remove obsolete .changes files from proposed-updates -# Copyright (C) 2001, 2002, 2003, 2004, 2006 James Troup +# Copyright (C) 2001, 2002, 2003, 2004, 2006, 2008 James Troup # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -80,11 +80,11 @@ def check_changes (filename): daklib.utils.fubar("unknown type, fix me") if not pu.has_key(pkg): # FIXME - daklib.utils.warn("%s doesn't seem to exist in p-u?? (from %s [%s])" % (pkg, file, filename)) + daklib.utils.warn("%s doesn't seem to exist in %s?? (from %s [%s])" % (pkg, Options["suite"], file, filename)) continue if not pu[pkg].has_key(arch): # FIXME - daklib.utils.warn("%s doesn't seem to exist for %s in p-u?? (from %s [%s])" % (pkg, arch, file, filename)) + daklib.utils.warn("%s doesn't seem to exist for %s in %s?? (from %s [%s])" % (pkg, arch, Options["suite"], file, filename)) continue pu_version = daklib.utils.re_no_epoch.sub('', pu[pkg][arch]) if pu_version == version: @@ -99,7 +99,8 @@ def check_changes (filename): if new_num_files == 0: print "%s: no files left, superseded by %s" % (filename, pu_version) dest = Cnf["Dir::Morgue"] + "/misc/" - daklib.utils.move(filename, dest) + if not Options["no-action"]: + daklib.utils.move(filename, dest) elif new_num_files < num_files: print "%s: lost files, MWAAP." % (filename) else: @@ -112,7 +113,7 @@ def check_joey (filename): file = daklib.utils.open_file(filename) cwd = os.getcwd() - os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"])) + os.chdir("%s/dists/%s" % (Cnf["Dir::Root"]), Options["suite"]) for line in file.readlines(): line = line.rstrip() @@ -139,13 +140,13 @@ def init_pu (): SELECT b.package, b.version, a.arch_string FROM bin_associations ba, binaries b, suite su, architecture a WHERE b.id = ba.bin AND ba.suite = su.id - AND su.suite_name = 'proposed-updates' AND a.id = b.architecture + AND su.suite_name = '%s' AND a.id = b.architecture UNION SELECT s.source, s.version, 'source' FROM src_associations sa, source s, suite su WHERE s.id = sa.source AND sa.suite = su.id - AND su.suite_name = 'proposed-updates' + AND su.suite_name = '%s' ORDER BY package, version, arch_string -""") +""" % (Options["suite"], Options["suite"])) ql = q.getresult() for i in ql: pkg = i[0] @@ -161,12 +162,18 @@ def main (): Cnf = daklib.utils.get_conf() Arguments = [('d', "debug", "Clean-Proposed-Updates::Options::Debug"), - ('v',"verbose","Clean-Proposed-Updates::Options::Verbose"), - ('h',"help","Clean-Proposed-Updates::Options::Help")] - for i in [ "debug", "verbose", "help" ]: + ('v', "verbose", "Clean-Proposed-Updates::Options::Verbose"), + ('h', "help", "Clean-Proposed-Updates::Options::Help"), + ('s', "suite", "Clean-Proposed-Updates::Options::Suite", "HasArg"), + ('n', "no-action", "Clean-Proposed-Updates::Options::No-Action"),] + for i in [ "debug", "verbose", "help", "no-action" ]: if not Cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)): Cnf["Clean-Proposed-Updates::Options::%s" % (i)] = "" + # suite defaults to proposed-updates + if not Cnf.has_key("Clean-Proposed-Updates::Options::Suite"): + Cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates" + arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) Options = Cnf.SubTree("Clean-Proposed-Updates::Options") diff --git a/dak/dak.py b/dak/dak.py index 10da0411..0eeb9d7b 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -29,7 +29,7 @@ ################################################################################ import sys, imp -import daklib.utils +import daklib.utils, daklib.extensions ################################################################################ @@ -91,7 +91,9 @@ def init(): "Clean cruft from incoming"), ("clean-proposed-updates", "Remove obsolete .changes from proposed-updates"), - + + ("transitions", + "Manage the release transition file"), ("check-overrides", "Override cruft checks"), ("check-proposed-updates", @@ -209,6 +211,8 @@ def main(): module.dak_userext = userext userext.dak_module = module + + daklib.extensions.init(cmdname, module, userext) if userext.init is not None: userext.init(cmdname) module.main() diff --git a/dak/import_keyring.py b/dak/import_keyring.py index 7f35b146..be35a5c2 100755 --- a/dak/import_keyring.py +++ b/dak/import_keyring.py @@ -237,7 +237,7 @@ def main(): name = desuid_byid[id][1] oname = db_uid_byid[id][1] if name and oname != name: - changes.append((uid[1], "Full name: %s\n" % (name))) + changes.append((uid[1], "Full name: %s" % (name))) projectB.query("UPDATE uid SET name = '%s' WHERE id = %s" % (pg.escape_string(name), id)) @@ -262,7 +262,7 @@ def main(): for f,(u,fid,kr) in db_fin_info.iteritems(): if kr != keyring_id: continue if f in fpr: continue - changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s\n" % (f))) + changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s" % (f))) projectB.query("UPDATE fingerprint SET keyring = NULL WHERE id = %d" % (fid)) # For the keys in this keyring, add/update any fingerprints that've @@ -275,7 +275,7 @@ def main(): if olduid == None: olduid = -1 if oldkid == None: oldkid = -1 if oldfid == -1: - changes.append((newuiduid, "Added key: %s\n" % (f))) + changes.append((newuiduid, "Added key: %s" % (f))) if newuid: projectB.query("INSERT INTO fingerprint (fingerprint, uid, keyring) VALUES ('%s', %d, %d)" % (f, newuid, keyring_id)) else: @@ -283,9 +283,11 @@ def main(): else: if newuid and olduid != newuid: if olduid != -1: - changes.append((newuiduid, "Linked key: %s (formerly belonging to %s)" % (f, db_uid_byid[olduid][0]))) + changes.append((newuiduid, "Linked key: %s" % f)) + changes.append((newuiduid, " (formerly belonging to %s)" % (db_uid_byid[olduid][0]))) else: - changes.append((newuiduid, "Linked key: %s (formerly unowned)\n" % (f))) + changes.append((newuiduid, "Linked key: %s" % f)) + changes.append((newuiduid, " (formerly unowned)")) projectB.query("UPDATE fingerprint SET uid = %d WHERE id = %d" % (newuid, oldfid)) if oldkid != keyring_id: @@ -298,12 +300,12 @@ def main(): changesd = {} for (k, v) in changes: if k not in changesd: changesd[k] = "" - changesd[k] += " " + v + changesd[k] += " %s\n" % (v) keys = changesd.keys() keys.sort() for k in keys: - print "%s\n%s" % (k, changesd[k]) + print "%s\n%s\n" % (k, changesd[k]) ################################################################################ diff --git a/dak/process_accepted.py b/dak/process_accepted.py index 20aab495..90edaf55 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -313,12 +313,17 @@ def install (): if dsc.get("dm-upload-allowed", "no") == "yes": uploader_ids = [maintainer_id] if dsc.has_key("uploaders"): - for u in dsc["uploaders"].split(","): - u = u.replace("'", "\\'") - u = u.strip() + for u in dsc["uploaders"].split(","): + u = u.replace("'", "\\'") + u = u.strip() uploader_ids.append( - daklib.database.get_or_set_maintainer_id(u)) + daklib.database.get_or_set_maintainer_id(u)) + added_ids = {} for u in uploader_ids: + if added_ids.has_key(u): + daklib.utils.warn("Already saw uploader %s for source %s" % (u, package)) + continue + added_ids[u]=1 projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u)) diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 4a4cfd6b..30737ed1 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -899,40 +899,77 @@ def check_urgency (): ################################################################################ -def check_md5sums (): - for file in files.keys(): +def check_hashes (): + # Make sure we recognise the format of the Files: field + format = changes.get("format", "0.0").split(".",1) + if len(format) == 2: + format = int(format[0]), int(format[1]) + else: + format = int(float(format[0])), 0 + + check_hash(".changes", files, "md5sum", apt_pkg.md5sum) + check_hash(".dsc", dsc_files, "md5sum", apt_pkg.md5sum) + + if format >= (1,8): + hashes = [("sha1", apt_pkg.sha1sum), + ("sha256", apt_pkg.sha256sum)] + else: + hashes = [] + + for x in changes: + if x.startswith("checksum-"): + h = x.split("-",1)[1] + if h not in dict(hashes): + reject("Unsupported checksum field in .changes" % (h)) + + for x in dsc: + if x.startswith("checksum-"): + h = x.split("-",1)[1] + if h not in dict(hashes): + reject("Unsupported checksum field in .dsc" % (h)) + + for h,f in hashes: try: - file_handle = daklib.utils.open_file(file) - except daklib.utils.cant_open_exc: - continue + fs = daklib.utils.build_file_list(changes, 0, "checksums-%s" % h, h) + check_hash(".changes %s" % (h), fs, h, f, files) + except daklib.utils.no_files_exc: + reject("No Checksums-%s: field in .changes file" % (h)) - # Check md5sum - if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]: - reject("%s: md5sum check failed." % (file)) - file_handle.close() - # Check size - actual_size = os.stat(file)[stat.ST_SIZE] - size = int(files[file]["size"]) - if size != actual_size: - reject("%s: actual file size (%s) does not match size (%s) in .changes" - % (file, actual_size, size)) + if "source" not in changes["architecture"]: continue + + try: + fs = daklib.utils.build_file_list(dsc, 1, "checksums-%s" % h, h) + check_hash(".dsc %s" % (h), fs, h, f, dsc_files) + except daklib.utils.no_files_exc: + reject("No Checksums-%s: field in .changes file" % (h)) + +################################################################################ + +def check_hash (where, files, key, testfn, basedict = None): + if basedict: + for file in basedict.keys(): + if file not in files: + reject("%s: no %s checksum" % (file, key)) + + for file in files.keys(): + if basedict and file not in basedict: + reject("%s: extraneous entry in %s checksums" % (file, key)) - for file in dsc_files.keys(): try: file_handle = daklib.utils.open_file(file) except daklib.utils.cant_open_exc: continue - # Check md5sum - if apt_pkg.md5sum(file_handle) != dsc_files[file]["md5sum"]: - reject("%s: md5sum check failed." % (file)) + # Check hash + if testfn(file_handle) != files[file][key]: + reject("%s: %s check failed." % (file, key)) file_handle.close() # Check size actual_size = os.stat(file)[stat.ST_SIZE] - size = int(dsc_files[file]["size"]) + size = int(files[file]["size"]) if size != actual_size: - reject("%s: actual file size (%s) does not match size (%s) in .dsc" - % (file, actual_size, size)) + reject("%s: actual file size (%s) does not match size (%s) in %s" + % (file, actual_size, size, where)) ################################################################################ @@ -1518,7 +1555,7 @@ def process_it (changes_file): valid_dsc_p = check_dsc() if valid_dsc_p: check_source() - check_md5sums() + check_hashes() check_urgency() check_timestamps() check_signed_by_key() diff --git a/dak/transitions.py b/dak/transitions.py new file mode 100755 index 00000000..e7cb99e8 --- /dev/null +++ b/dak/transitions.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python + +# Display, edit and check the release manager's transition file. +# Copyright (C) 2008 Joerg Jaspert + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +# if klecker.d.o died, I swear to god, I'm going to migrate to gentoo. + +################################################################################ + +import os, pg, sys, time, errno, fcntl, tempfile, pwd, re +import apt_pkg +import daklib.database +import daklib.utils +import syck + +# Globals +Cnf = None +Options = None +projectB = None + +re_broken_package = re.compile(r"[a-zA-Z]\w+\s+\-.*") + +################################################################################ + +##################################### +#### This may run within sudo !! #### +##################################### +def init(): + global Cnf, Options, projectB + + apt_pkg.init() + + Cnf = daklib.utils.get_conf() + + Arguments = [('h',"help","Edit-Transitions::Options::Help"), + ('e',"edit","Edit-Transitions::Options::Edit"), + ('i',"import","Edit-Transitions::Options::Import", "HasArg"), + ('c',"check","Edit-Transitions::Options::Check"), + ('s',"sudo","Edit-Transitions::Options::Sudo"), + ('n',"no-action","Edit-Transitions::Options::No-Action")] + + for i in ["help", "no-action", "edit", "import", "check", "sudo"]: + if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)): + Cnf["Edit-Transitions::Options::%s" % (i)] = "" + + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) + + Options = Cnf.SubTree("Edit-Transitions::Options") + + if Options["help"]: + usage() + + whoami = os.getuid() + whoamifull = pwd.getpwuid(whoami) + username = whoamifull[0] + if username != "dak": + print "Non-dak user: %s" % username + Options["sudo"] = "y" + + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) + daklib.database.init(Cnf, projectB) + +################################################################################ + +def usage (exit_code=0): + print """Usage: transitions [OPTION]... +Update and check the release managers transition file. + +Options: + + -h, --help show this help and exit. + -e, --edit edit the transitions file + -i, --import check and import transitions from file + -c, --check check the transitions file, remove outdated entries + -S, --sudo use sudo to update transitions file + -n, --no-action don't do anything (only affects check)""" + + sys.exit(exit_code) + +################################################################################ + +##################################### +#### This may run within sudo !! #### +##################################### +def load_transitions(trans_file): + # Parse the yaml file + sourcefile = file(trans_file, 'r') + sourcecontent = sourcefile.read() + failure = False + try: + trans = syck.load(sourcecontent) + except syck.error, msg: + # Someone fucked it up + print "ERROR: %s" % (msg) + return None + + # lets do further validation here + checkkeys = ["source", "reason", "packages", "new", "rm"] + + # If we get an empty definition - we just have nothing to check, no transitions defined + if type(trans) != dict: + # This can be anything. We could have no transitions defined. Or someone totally fucked up the + # file, adding stuff in a way we dont know or want. Then we set it empty - and simply have no + # transitions anymore. User will see it in the information display after he quit the editor and + # could fix it + trans = "" + return trans + + try: + for test in trans: + t = trans[test] + + # First check if we know all the keys for the transition and if they have + # the right type (and for the packages also if the list has the right types + # included, ie. not a list in list, but only str in the list) + for key in t: + if key not in checkkeys: + print "ERROR: Unknown key %s in transition %s" % (key, test) + failure = True + + if key == "packages": + if type(t[key]) != list: + print "ERROR: Unknown type %s for packages in transition %s." % (type(t[key]), test) + failure = True + try: + for package in t["packages"]: + if type(package) != str: + print "ERROR: Packages list contains invalid type %s (as %s) in transition %s" % (type(package), package, test) + failure = True + if re_broken_package.match(package): + # Someone had a space too much (or not enough), we have something looking like + # "package1 - package2" now. + print "ERROR: Invalid indentation of package list in transition %s, around package(s): %s" % (test, package) + failure = True + except TypeError: + # In case someone has an empty packages list + print "ERROR: No packages defined in transition %s" % (test) + failure = True + continue + + elif type(t[key]) != str: + if key == "new" and type(t[key]) == int: + # Ok, debian native version + continue + else: + print "ERROR: Unknown type %s for key %s in transition %s" % (type(t[key]), key, test) + failure = True + + # And now the other way round - are all our keys defined? + for key in checkkeys: + if key not in t: + print "ERROR: Missing key %s in transition %s" % (key, test) + failure = True + except TypeError: + # In case someone defined very broken things + print "ERROR: Unable to parse the file" + failure = True + + + if failure: + return None + + return trans + +################################################################################ + +##################################### +#### This may run within sudo !! #### +##################################### +def lock_file(file): + for retry in range(10): + lock_fd = os.open(file, os.O_RDWR | os.O_CREAT) + try: + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_fd + except OSError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': + print "Unable to get lock for %s (try %d of 10)" % \ + (file, retry+1) + time.sleep(60) + else: + raise + + daklib.utils.fubar("Couldn't obtain lock for %s." % (lockfile)) + +################################################################################ + +##################################### +#### This may run within sudo !! #### +##################################### +def write_transitions(from_trans): + """Update the active transitions file safely. + This function takes a parsed input file (which avoids invalid + files or files that may be be modified while the function is + active), and ensure the transitions file is updated atomically + to avoid locks.""" + + trans_file = Cnf["Dinstall::Reject::ReleaseTransitions"] + trans_temp = trans_file + ".tmp" + + trans_lock = lock_file(trans_file) + temp_lock = lock_file(trans_temp) + + destfile = file(trans_temp, 'w') + syck.dump(from_trans, destfile) + destfile.close() + + os.rename(trans_temp, trans_file) + os.close(temp_lock) + os.close(trans_lock) + +################################################################################ + +class ParseException(Exception): + pass + +########################################## +#### This usually runs within sudo !! #### +########################################## +def write_transitions_from_file(from_file): + """We have a file we think is valid; if we're using sudo, we invoke it + here, otherwise we just parse the file and call write_transitions""" + + # Lets check if from_file is in the directory we expect it to be in + if not os.path.abspath(from_file).startswith(Cnf["Transitions::TempPath"]): + print "Will not accept transitions file outside of %s" % (Cnf["Transitions::TempPath"]) + sys.exit(3) + + if Options["sudo"]: + os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H", + "/usr/local/bin/dak", "transitions", "--import", from_file) + else: + trans = load_transitions(from_file) + if trans is None: + raise ParseException, "Unparsable transitions file %s" % (file) + write_transitions(trans) + +################################################################################ + +def temp_transitions_file(transitions): + # NB: file is unlinked by caller, but fd is never actually closed. + # We need the chmod, as the file is (most possibly) copied from a + # sudo-ed script and would be unreadable if it has default mkstemp mode + + (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Transitions::TempPath"]) + os.chmod(path, 0644) + f = open(path, "w") + syck.dump(transitions, f) + return path + +################################################################################ + +def edit_transitions(): + trans_file = Cnf["Dinstall::Reject::ReleaseTransitions"] + edit_file = temp_transitions_file(load_transitions(trans_file)) + + editor = os.environ.get("EDITOR", "vi") + + while True: + result = os.system("%s %s" % (editor, edit_file)) + if result != 0: + os.unlink(edit_file) + daklib.utils.fubar("%s invocation failed for %s, not removing tempfile." % (editor, edit_file)) + + # Now try to load the new file + test = load_transitions(edit_file) + + if test == None: + # Edit is broken + print "Edit was unparsable." + prompt = "[E]dit again, Drop changes?" + default = "E" + else: + print "Edit looks okay.\n" + print "The following transitions are defined:" + print "------------------------------------------------------------------------" + transition_info(test) + + prompt = "[S]ave, Edit again, Drop changes?" + default = "S" + + answer = "XXX" + while prompt.find(answer) == -1: + answer = daklib.utils.our_raw_input(prompt) + if answer == "": + answer = default + answer = answer[:1].upper() + + if answer == 'E': + continue + elif answer == 'D': + os.unlink(edit_file) + print "OK, discarding changes" + sys.exit(0) + elif answer == 'S': + # Ready to save + break + else: + print "You pressed something you shouldn't have :(" + sys.exit(1) + + # We seem to be done and also have a working file. Copy over. + write_transitions_from_file(edit_file) + os.unlink(edit_file) + + print "Transitions file updated." + +################################################################################ + +def check_transitions(transitions): + to_dump = 0 + to_remove = [] + # Now look through all defined transitions + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + + print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + + if current == None: + # No package in testing + print "Transition source %s not in testing, transition still ongoing." % (source) + else: + compare = apt_pkg.VersionCompare(current, expected) + if compare < 0: + # This is still valid, the current version in database is older than + # the new version we wait for + print "This transition is still ongoing, we currently have version %s" % (current) + else: + print "REMOVE: This transition is over, the target package reached testing. REMOVE" + print "%s wanted version: %s, has %s" % (source, expected, current) + to_remove.append(trans) + to_dump = 1 + print "-------------------------------------------------------------------------" + + if to_dump: + prompt = "Removing: " + for remove in to_remove: + prompt += remove + prompt += "," + + prompt += " Commit Changes? (y/N)" + answer = "" + + if Options["no-action"]: + answer="n" + else: + answer = daklib.utils.our_raw_input(prompt).lower() + + if answer == "": + answer = "n" + + if answer == 'n': + print "Not committing changes" + sys.exit(0) + elif answer == 'y': + print "Committing" + for remove in to_remove: + del transitions[remove] + + edit_file = temp_transitions_file(transitions) + write_transitions_from_file(edit_file) + + print "Done" + else: + print "WTF are you typing?" + sys.exit(0) + +################################################################################ + +def print_info(trans, source, expected, rm, reason, packages): + print """Looking at transition: %s + Source: %s + New Version: %s + Responsible: %s + Description: %s + Blocked Packages (total: %d): %s +""" % (trans, source, expected, rm, reason, len(packages), ", ".join(packages)) + return + +################################################################################ + +def transition_info(transitions): + for trans in transitions: + t = transitions[trans] + source = t["source"] + expected = t["new"] + + # Will be None if nothing is in testing. + current = daklib.database.get_suite_version(source, "testing") + + print_info(trans, source, expected, t["rm"], t["reason"], t["packages"]) + + if current == None: + # No package in testing + print "Transition source %s not in testing, transition still ongoing." % (source) + else: + compare = apt_pkg.VersionCompare(current, expected) + print "Apt compare says: %s" % (compare) + if compare < 0: + # This is still valid, the current version in database is older than + # the new version we wait for + print "This transition is still ongoing, we currently have version %s" % (current) + else: + print "This transition is over, the target package reached testing, should be removed" + print "%s wanted version: %s, has %s" % (source, expected, current) + print "-------------------------------------------------------------------------" + +################################################################################ + +def main(): + global Cnf + + ##################################### + #### This can run within sudo !! #### + ##################################### + init() + + # Check if there is a file defined (and existant) + transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "") + if transpath == "": + daklib.utils.warn("Dinstall::Reject::ReleaseTransitions not defined") + sys.exit(1) + if not os.path.exists(transpath): + daklib.utils.warn("ReleaseTransitions file, %s, not found." % + (Cnf["Dinstall::Reject::ReleaseTransitions"])) + sys.exit(1) + # Also check if our temp directory is defined and existant + temppath = Cnf.get("Transitions::TempPath", "") + if temppath == "": + daklib.utils.warn("Transitions::TempPath not defined") + sys.exit(1) + if not os.path.exists(temppath): + daklib.utils.warn("Temporary path %s not found." % + (Cnf["Transitions::TempPath"])) + sys.exit(1) + + if Options["import"]: + try: + write_transitions_from_file(Options["import"]) + except ParseException, m: + print m + sys.exit(2) + sys.exit(0) + ############################################## + #### Up to here it can run within sudo !! #### + ############################################## + + # Parse the yaml file + transitions = load_transitions(transpath) + if transitions == None: + # Something very broken with the transitions, exit + daklib.utils.warn("Could not parse existing transitions file. Aborting.") + sys.exit(2) + + if Options["edit"]: + # Let's edit the transitions file + edit_transitions() + elif Options["check"]: + # Check and remove outdated transitions + check_transitions(transitions) + else: + # Output information about the currently defined transitions. + print "Currently defined transitions:" + transition_info(transitions) + + sys.exit(0) + +################################################################################ + +if __name__ == '__main__': + main() diff --git a/daklib/database.py b/daklib/database.py old mode 100644 new mode 100755 index a40696e2..5c362604 --- a/daklib/database.py +++ b/daklib/database.py @@ -41,6 +41,7 @@ maintainer_cache = {} fingerprint_id_cache = {} queue_id_cache = {} uid_id_cache = {} +suite_version_cache = {} ################################################################################ @@ -223,6 +224,29 @@ def get_source_id (source, version): return source_id +def get_suite_version(source, suite): + global suite_version_cache + cache_key = "%s_%s" % (source, suite) + + if suite_version_cache.has_key(cache_key): + return suite_version_cache[cache_key] + + q = projectB.query(""" + SELECT s.version FROM source s, suite su, src_associations sa + WHERE sa.source=s.id + AND sa.suite=su.id + AND su.suite_name='%s' + AND s.source='%s'""" + % (suite, source)) + + if not q.getresult(): + return None + + version = q.getresult()[0][0] + suite_version_cache[cache_key] = version + + return version + ################################################################################ def get_or_set_maintainer_id (maintainer): diff --git a/daklib/extensions.py b/daklib/extensions.py new file mode 100644 index 00000000..d5da89d8 --- /dev/null +++ b/daklib/extensions.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Utility functions for extensions +# Copyright (C) 2008 Anthony Towns + +################################################################################ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +dak_functions_to_replace = {} +dak_replaced_functions = {} + +def replace_dak_function(module,name): + """Decorator to make a function replace a standard dak function + in a given module. The replaced function will be provided as + the first argument.""" + + def x(f): + def myfunc(*a,**kw): + global replaced_funcs + f(dak_replaced_functions[name], *a, **kw) + myfunc.__name__ = f.__name__ + myfunc.__doc__ = f.__doc__ + myfunc.__dict__.update(f.__dict__) + + fnname = "%s:%s" % (module, name) + if fnname in dak_functions_to_replace: + raise Exception, \ + "%s in %s already marked to be replaced" % (name, module) + dak_functions_to_replace["%s:%s" % (module,name)] = myfunc + return f + return x + +################################################################################ + +def init(name, module, userext): + global dak_replaced_functions + + # This bit should be done automatically too + dak_replaced_functions = {} + for f,newfunc in dak_functions_to_replace.iteritems(): + m,f = f.split(":",1) + if len(f) > 0 and m == name: + dak_replaced_functions[f] = module.__dict__[f] + module.__dict__[f] = newfunc + + diff --git a/daklib/utils.py b/daklib/utils.py index 665d9928..16cc1ff1 100755 --- a/daklib/utils.py +++ b/daklib/utils.py @@ -43,6 +43,7 @@ re_taint_free = re.compile(r"^[-+~/\.\w]+$") re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>") re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$") +re_verwithext = re.compile(r"^(\d+)(?:\.(\d+))(?:\s+\((\S+)\))?$") changes_parse_error_exc = "Can't parse line in .changes file" invalid_dsc_format_exc = "Invalid .dsc file" @@ -229,31 +230,48 @@ The rules for (signing_rules == 1)-mode are: # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl -def build_file_list(changes, is_a_dsc=0): +def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"): files = {} # Make sure we have a Files: field to parse... - if not changes.has_key("files"): - raise no_files_exc + if not changes.has_key(field): + raise no_files_exc # Make sure we recognise the format of the Files: field - format = changes.get("format", "") - if format != "": - format = float(format) - if not is_a_dsc and (format < 1.5 or format > 2.0): - raise nk_format_exc, format + format = re_verwithext.search(changes.get("format", "0.0")) + if not format: + raise nk_format_exc, "%s" % (changes.get("format","0.0")) + + format = format.groups() + if format[1] == None: + format = int(float(format[0])), 0, format[2] + else: + format = int(format[0]), int(format[1]), format[2] + if format[2] == None: + format = format[:2] + + if is_a_dsc: + if format != (1,0): + raise nk_format_exc, "%s" % (changes.get("format","0.0")) + else: + if (format < (1,5) or format > (1,8)): + raise nk_format_exc, "%s" % (changes.get("format","0.0")) + if field != "files" and format < (1,8): + raise nk_format_exc, "%s" % (changes.get("format","0.0")) + + includes_section = (not is_a_dsc) and field == "files" # Parse each entry/line: - for i in changes["files"].split('\n'): + for i in changes[field].split('\n'): if not i: break s = i.split() section = priority = "" try: - if is_a_dsc: - (md5, size, name) = s - else: + if includes_section: (md5, size, section, priority, name) = s + else: + (md5, size, name) = s except ValueError: raise changes_parse_error_exc, i @@ -264,8 +282,9 @@ def build_file_list(changes, is_a_dsc=0): (section, component) = extract_component_from_section(section) - files[name] = Dict(md5sum=md5, size=size, section=section, + files[name] = Dict(size=size, section=section, priority=priority, component=component) + files[name][hashname] = md5 return files diff --git a/debian/control b/debian/control index 97c1fbe1..2d6678b9 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Standards-Version: 3.5.6.0 Package: dak Architecture: any -Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev +Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev, python-syck (>= 0.61.2-1) Suggests: lintian, linda, less, binutils-multiarch, symlinks, postgresql (>= 7.1.0), dsync Description: Debian's archive maintenance scripts This is a collection of archive maintenance scripts used by the diff --git a/docs/README.stable-point-release b/docs/README.stable-point-release index 34059ef9..176b33eb 100644 --- a/docs/README.stable-point-release +++ b/docs/README.stable-point-release @@ -13,25 +13,40 @@ o Install, reject and remove packages as directed by the SRM using NB: removing packages are not logged to the stable ChangeLog; you need to do that byhand. -o Do anything in proposed-updates/TODO +o If you installed a debian-installer upload; migrate the relevant + installer-*/$release directory from proposed-updates to stable. + (Including potentially removing older versions) + +o Decruft stable in coordination with SRMs + +o Do anything in proposed-updates/TODO o Close any applicable stable bugs (hint: http://bugs.debian.org/cgi-bin/pkgreport.cgi?pkg=ftp.debian.org&include=etch) -o Update version number in README, README.html and dists/README (ftp-master only) +o Update version number in README, README.html and dists/README o Update the 'Debian.r' symlink in dists/ o Clean up dists/stable/ChangeLog (add header, basically) o Update version fields in dak.conf o Update fields in suite table in postgresql (see below) -o Comment out "Untouchable" in dak.conf -o Run 'dak make-suite-file-list -s stable' +o Run 'dak make-suite-file-list --force -s stable' o Run apt-ftparchive generate apt.conf.stable -o Run 'dak generate-releases stable' ** FIXME: requires apt.conf.stable stanza for stable in apt.conf - ** FIXME: must be run as dak -o Uncomment "Untouchable" in dak.conf +o Run 'dak generate-releases --force-touch --apt-conf apt.conf.stable stable' -Yes, this sucks and more of it should be automated. +[Yes, this sucks and more of it should be automated. c.f. ~ajt/pointupdate] ####################################################### -update suite set version = '4.0r1' where suite_name = 'stable'; -update suite set description = 'Debian 4.0r1 Released 15th August 2007' where suite_name = 'stable'; +update suite set version = '4.0r3' where suite_name = 'stable'; +update suite set description = 'Debian 4.0r3 Released 16th February 2008' where suite_name = 'stable'; + +Rough Guide to doing Old-Stable Point Releases in Debian +-------------------------------------------------------- + +Pretty much as above, except that process-accepted doesn't know about +oldstable, so you have to do some surgery on it first to make it +support that. Probably want to disable cron.daily whilst doing so. +Also watch out for the installing_to_stable detection which doesn't +work well with the current layout of oldstable-proposed-updates (as a +symlink to $distro-proposed-updates). clean-proposed-updates, +cruft-report and most everything else support a -s/--suite so they +sould be fine to use. diff --git a/docs/transitions.txt b/docs/transitions.txt new file mode 100644 index 00000000..5b52bb31 --- /dev/null +++ b/docs/transitions.txt @@ -0,0 +1,275 @@ +Contents: + +1. Little "Howto Use it" +2. Explanation of how it works + + +1. Little "Howto Use it" +------------------------ + +The input file is in YAML format. Do bnot bother with comments, they +will be removed. + +The format: Dont use tabs for indentation, use spaces. + +Strings should be within "", but normally work without. +Exception: Version-numbers with an epoch really do want to be in +"". YES, THEY WANT TO (or they get interpreted in a way you dont expect +it). + +Keys (The order of them does not matter, only the indentation): + +short_tag: A short tag for the transition, like apt_update + reason: One-line reason what is intended with it + source: Source package that needs to transition + new: New version of the target package + rm: Name of the Release Team member responsible for this transition + packages: Array of package names that are affected by this transition + + +The following example wants to +a.) update apt to version 0.7.12, the responsible Release Team member +is Andreas Barth, and it affects some apt related packages and +b.) wants to do something similar for lintian. + +apt_update: + packages: + - apt + - synaptic + - cron-apt + - debtags + - feta + - apticron + - aptitude + reason: "Apt needs to transition to testing to get foo and bar done" + source: apt + new: 0.7.12 + rm: Andreas Barth +lintian_breakage: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck + + +######################################################################## +######################################################################## + + +2. Explanation of how it works +------------------------------ + +Assume the following transition is defined: + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck + +Also assume the lintian situation on this archive looks like this: + lintian | 1.23.28~bpo.1 | sarge-backports | source, all + lintian | 1.23.45~bpo40+1 | etch-backports | source, all + +------------------------------------------------------------------------ + +Now, I try to upload a (NEW, but that makes no difference) version of +python-syck: + +$ dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +REJECT +Rejected: python-syck: part of the lintian_funtest transition. + +Your package is part of a testing transition designed to get lintian migrated +(it currently is at version 1.23.28~bpo.1, we need version 1.23.45~bpo40+1) + +Transition description: Testing a new feature + +This transition is managed by the Release Team, and Ganneff +is the Release-Team member responsible for it. +Please contact Ganneff or debian-release@lists.debian.org if you +need further assistance. + +------------------------------------------------------------------------ + +Lets change the definition of the transition, assume it is now: + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.22.28~bpo.1 + rm: Ganneff + packages: + - lintian + - python-syck + +Which checks for a version older than the version actually available. Result: + +dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +NEW for etch-backports +(new) python-syck_0.61.2-1~bpo40+1.diff.gz extra python +(new) python-syck_0.61.2-1~bpo40+1.dsc extra python +(new) python-syck_0.61.2-1~bpo40+1_i386.deb extra python +PySyck python bindings to the Syck YAML parser kit + Syck is a simple YAML parser kit. + . +[...] the whole stuff about a new package. + +------------------------------------------------------------------------ + +For completness, change the transition to (exact existing version): +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.28~bpo.1 + rm: Ganneff + packages: + - lintian + +and the result is: + +dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes + +python-syck_0.61.2-1~bpo40+1_i386.changes +NEW for etch-backports +[... we know this ...] + +------------------------------------------------------------------------ + +The second part is the check_transitions script. +For that we take the following transitions as example: + +apt_update: + reason: "Apt needs to transition to testing to get foo and bar done" + source: apt + new: 0.2.12-1+b1.3 + rm: Andreas Barth + packages: + - apt + - synaptic + - cron-apt + - debtags + - feta + - apticron + - aptitude +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.23.45~bpo40+1 + rm: Ganneff + packages: + - lintian + - python-syck +bar_breaks_it: + reason: We dont want bar to break it + source: bar + new: "9:99" + rm: Ganneff + packages: + - kdelibs + - qt4-x11 + - libqt-perl + +Running check-transitions ends up with the following output: + +Looking at transition: lintian_funtest + Source: lintian + New Version: 1.23.45~bpo40+1 + Responsible: Ganneff + Description: Testing a new feature + Blocked Packages (total: 2): lintian, python-syck + +Apt compare says: -2 +This transition is still ongoing, we currently have version 1.23.28~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: apt_update + Source: apt + New Version: 0.2.12-1+b1.3 + Responsible: Andreas Barth + Description: Apt needs to transition to testing to get foo and bar done + Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude + +Apt compare says: 4 +This transition is over, the target package reached testing, removing +apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: bar_breaks_it + Source: bar + New Version: 9:99 + Responsible: Ganneff + Description: We dont want bar to break it + Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl + +Transition source bar not in testing, transition still ongoing. +------------------------------------------------------------------------- +I: I would remove the apt_update transition + + +Changing our transition definitions for lintian (keeping the rest as +above) to + +lintian_funtest: + reason: "Testing a new feature" + source: lintian + new: 1.22.28~bpo.1 + rm: Ganneff + packages: + - lintian + - python-syck + +now we get + +Looking at transition: lintian_funtest + Source: lintian + New Version: 1.22.28~bpo.1 + Responsible: Ganneff + Description: Testing a new feature + Blocked Packages (total: 2): lintian, python-syck + +Apt compare says: 1 +This transition is over, the target package reached testing, removing +lintian wanted version: 1.22.28~bpo.1, has 1.23.28~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: apt_update + Source: apt + New Version: 0.2.12-1+b1.3 + Responsible: Andreas Barth + Description: Apt needs to transition to testing to get foo and bar done + Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude + +Apt compare says: 4 +This transition is over, the target package reached testing, removing +apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1 +------------------------------------------------------------------------- + +Looking at transition: bar_breaks_it + Source: bar + New Version: 9:99 + Responsible: Ganneff + Description: We dont want bar to break it + Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl + +Transition source bar not in testing, transition still ongoing. +------------------------------------------------------------------------- +I: I would remove the lintian_funtest transition +I: I would remove the apt_update transition + + +Not using the -n switch would turn the I: in actual removals :) +The check-transition command is meant for the release team to always run +it when they change a transition definition. It checks if the yaml is +valid and can be loaded (but if not the archive simply does no reject) +and also shows a nice overview. diff --git a/scripts/debian/byhand-di b/scripts/debian/byhand-di new file mode 100755 index 00000000..0a004f38 --- /dev/null +++ b/scripts/debian/byhand-di @@ -0,0 +1,101 @@ +#!/bin/sh -ue + +if [ $# -lt 4 ]; then + echo "Usage: $0 filename version arch changes_file" + exit 1 +fi + +TARBALL="$1" # Tarball to read, compressed with gzip +VERSION="$2" +ARCH="$3" +CHANGES="$4" # Changes file for the upload + +error() { + echo "$*" + exit 1 +} + +# Check validity of version number +# Expected are: YYYYMMDD, YYYYMMDD.x, YYYYMMDDx +if ! echo "$VERSION" | grep -Eq "^[0-9]{8}(|(\.|[a-z]+)[0-9]+)$"; then + error "Invalid version: '$VERSION'" +fi + +# Get the target suite from the Changes file +# NOTE: it may be better to pass this to the script as a parameter! +SUITE="$(grep "^Distribution:" "$CHANGES" | awk '{print $2}')" +case $SUITE in + "") + error "Error: unable to determine suite from Changes file" + ;; + unstable|sid) + : # nothing to do + ;; + *) + SUITE="${SUITE}-proposed-updates" + ;; +esac + +# This must end with / +TARGET="/srv/ftp.debian.org/ftp/dists/$SUITE/main/installer-$ARCH/" + +# Check validity of the target directory +# This could fail, for example for new architectures; doing +# a regular BYHAND is safer in that case +if [ ! -d "$TARGET" ]; then + mkdir -p "$TARGET" +fi +# Check that there isn't already a directory for this version +if [ -d "$TARGET/$VERSION" ]; then + error "Directory already exists: $TARGET/$VERSION" +fi + +# We know all data to be in ./installer-/; see if there's +# anything else in the tarball except that and the 'current' symlink +if tar tzf "$TARBALL" | \ + grep -Eqv "^\./(installer-$ARCH/($VERSION/.*|current|)|)$"; then + error "Tarball contains unexpected contents" +fi + +# Create a temporary directory where to store the images +umask 002 +TMPDIR="$(mktemp -td byhand-di.XXXXXX)" + +# If we fail somewhere, cleanup the temporary directory +cleanup() { + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +# Extract the data into the temporary directory +tar xzf "$TARBALL" --directory="$TMPDIR" "./installer-$ARCH/" + +# Check the 'current' symlink +if [ ! -L $TMPDIR/installer-$ARCH/current ]; then + error "Missing 'current' symlink" +elif [ X"$(readlink "$TMPDIR/installer-$ARCH/current")" != X"$VERSION" ]; then + error "Incorrect 'current' symlink" +fi + +# We should have an MD5SUMS file; use that for a final check +if [ -r "$TMPDIR/installer-$ARCH/$VERSION/images/MD5SUMS" ]; then + ( + cd "$TMPDIR/installer-$ARCH/$VERSION/images" + md5sum -c --status MD5SUMS || error "Error while checking MD5SUMS" + ) +else + error "Missing MD5SUMS file" +fi + +# Move the data to the final location +mv "$TMPDIR/installer-$ARCH/$VERSION" "$TARGET" +mv "$TMPDIR/installer-$ARCH/current" "$TARGET" + +# Fixup permissions +find "$TARGET/$VERSION" -type d -exec chmod 755 {} + +find "$TARGET/$VERSION" -type f -exec chmod 644 {} + + +trap - EXIT +cleanup + +exit 0 diff --git a/scripts/debian/expire_dumps b/scripts/debian/expire_dumps new file mode 100755 index 00000000..9fa6adeb --- /dev/null +++ b/scripts/debian/expire_dumps @@ -0,0 +1,143 @@ +#!/usr/bin/python + +# Copyright (C) 2007 Florian Reitmeir +# Copyright (C) 2008 Joerg Jaspert + +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# requires: python-dateutil + +import glob, os, sys +import time, datetime +import re +from datetime import datetime +from datetime import timedelta +from optparse import OptionParser + +RULES = [ + {'days':14, 'interval':0}, + {'days':31, 'interval':7}, + {'days':365, 'interval':31}, + {'days':3650, 'interval':365}, + + # keep 14 days, all each day + # keep 31 days, 1 each 7th day + # keep 365 days, 1 each 31th day +] + +TODAY = datetime.today() +VERBOSE = False +NOACTION = False +PRINT = False +PREFIX = '' +PATH = '' + +def all_files(pattern, search_path, pathsep=os.pathsep): + """ Given a search path, yield all files matching the pattern. """ + for path in search_path.split(pathsep): + for match in glob.glob(os.path.join(path, pattern)): + yield match + +def parse_file_dates(list): + out = [] + # dump_2006.05.02-11:52:01.bz2 + p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$') + for file in list: + m = p.search(file) + if m: + d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6))) + out.append({'name': file, 'date': d}) + return out + +def prepare_rules(rules): + out = [] + for rule in rules: + out.append( + { + 'days':timedelta(days=rule['days']), + 'interval':timedelta(days=rule['interval'])} + ) + return out + +def expire(rules, list): + t_rules=prepare_rules(rules) + rule = t_rules.pop(0) + last = list.pop(0) + + for file in list: + if VERBOSE: + print "current file to expire: " + file['name'] + print file['date'] + + # check if rule applies + if (file['date'] < (TODAY-rule['days'])): + if VERBOSE: + print "move to next rule" + if t_rules: + rule = t_rules.pop(0) + + if (last['date'] - file['date']) < rule['interval']: + if VERBOSE: + print "unlink file:" + file['name'] + if PRINT: + print file['name'] + if not NOACTION: + os.unlink(file['name']) + else: + last = file + if VERBOSE: + print "kept file:" + file['name'] + + +parser = OptionParser() +parser.add_option("-d", "--directory", dest="directory", + help="directory name", metavar="Name") +parser.add_option("-f", "--pattern", dest="pattern", + help="Pattern maybe some glob", metavar="*.backup") +parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, + help="verbose") +parser.add_option("-n", "--no-action", action="store_true", dest="noaction", default=False, + help="just prints what would be done, this implies verbose") +parser.add_option("-p", "--print", action="store_true", dest="printfiles", default=False, + help="just print the filenames that should be deleted, this forbids verbose") + +(options, args) = parser.parse_args() + +if (not options.directory): + parser.error("no directory to check given") + +if options.noaction: + VERBOSE=True + NOACTION=True + +if options.verbose: + VERBOSE=True + +if options.printfiles: + VERBOSE=False + PRINT=True + +files = sorted( list(all_files(options.pattern,options.directory)), reverse=True ); + +if not files: + sys.exit(0) + +files_dates = parse_file_dates(files); +expire(RULES, files_dates) diff --git a/setup/init_pool.sql b/setup/init_pool.sql index 8797acd5..0ab91ad6 100644 --- a/setup/init_pool.sql +++ b/setup/init_pool.sql @@ -184,3 +184,19 @@ CREATE INDEX binaries_maintainer ON binaries (maintainer); CREATE INDEX binaries_fingerprint on binaries (sig_fpr); CREATE INDEX source_fingerprint on source (sig_fpr); CREATE INDEX dsc_files_file ON dsc_files (file); + +-- Own function +CREATE FUNCTION space_concat(text, text) RETURNS text + AS $_$select case +WHEN $2 is null or $2 = '' THEN $1 +WHEN $1 is null or $1 = '' THEN $2 +ELSE $1 || ' ' || $2 +END$_$ + LANGUAGE sql; + +CREATE AGGREGATE space_separated_list ( + BASETYPE = text, + SFUNC = space_concat, + STYPE = text, + INITCOND = '' +);