+2008-04-22 Joerg Jaspert <joerg@debian.org>
+
+ * setup/init_pool.sql: added a function/aggregate for the release
+ team to base some script on it.
+
+ * config/debian/cron.daily: push katie@merkel to immediately start
+ the sync of projectb there.
+
+2008-04-21 Joerg Jaspert <joerg@debian.org>
+
+ * scripts/debian/expire_dumps: New script, expires old database
+ dumps, using a scheme to keep more of the recent dumps.
+
+ * config/debian/cron.daily: Use the new script. Also - compress
+ all files older than 7 days, instead of 30.
+
+ * dak/process_accepted.py (install): Do not break if a
+ source/maintainer combination is already in src_uploaders, "just"
+ warn us.
+
+2008-04-20 Thomas Viehmann <tviehmann@debian.org>
+
+ * daklib/utils.py (build_file_list): Deal with "Format 3 style"
+ Format lines (ie. those having extra text appended).
+
+2008-04-18 Joerg Jaspert <joerg@debian.org>
+
+ * config/debian/dak.conf: Add mapping stable-proposed-updates
+ -> proposed-updates.
+
+ * dak/transitions.py (load_transitions): Additionally check for
+ invalid package list indentation
+
+2008-04-17 Joerg Jaspert <joerg@debian.org>
+
+ * config/debian/dak.conf: Add TempPath statement for the Release
+ Transitions script
+
+ * dak/transitions.py (temp_transitions_file): Use the TempPath
+ (write_transitions_from_file): Check if the file we should get our
+ transitions from is in our TempPath, error out if it isnt
+ (main): Check for TempPath existance
+
+2008-04-12 James Troup <troup@debian.org>
+
+ * dak/clean_proposed_updates.py: add support for -s/--suite and
+ -n/--no-action.
+
+2008-04-11 Anthony Towns <ajt@debian.org>
+
+ * dak/utils.py: build_file_list() extra parameters so it can
+ build a file list for checksums-foo fields. Don't use float() to
+ compare formats, because Format: 1.10 should compare greater than
+ Format: 1.9 (use "1.9".split(".",1) and tuple comparison instead)
+
+ * dak/process_unchecked.py: check_md5sum becomes check_hashes
+ and check_hash. If changes format is 1.8 or later, also check
+ checksums-sha1 and checksums-sha256 for both .changes and .dsc,
+ and reject on presence/absence of un/expected checksums-* fields.
+
+2008-04-07 Joerg Jaspert <joerg@debian.org>
+
+ * daklib/utils.py (build_file_list): Check for dpkg .changes
+ adjusted to reject newer (and right now broken) 1.8 version, until
+ dpkg (or debsign) is fixed and doesn't produce invalid .changes anymore
+
+2008-03-22 Joerg Jaspert <joerg@debian.org>
+
+ * dak/transitions.py (load_transitions): Check if all our keys are
+ defined, if there are only keys defined we want and also the types
+ of the various keys.
+
+2008-03-22 Anthony Towns <ajt@debian.org>
+
+ * dak/edit_transitions.py: Add --import option.
+ Add --use-sudo option. Use fcntl locking for writing.
+ Move writing into a function (write_transitions).
+ Reinvoke self using sudo and --import if necessary.
+ Move temporary file creation into a function, use mkstemp.
+ Rename to "dak transitions".
+
+2008-03-21 Joerg Jaspert <joerg@debian.org>
+
+ * dak/edit_transitions.py (edit_transitions): Use sudo to copy the
+ edited file back in place
+ (check_transitions): Use proper locking and also use sudo to copy
+ the new file in place
+
+2008-03-21 Anthony Towns <ajt@debian.org>
+
+ * config/debian/extensions.py: Add infrastructure for replacing
+ functions in dak modules; add upload blocking for dpkg.
+
+2008-03-12 Joerg Jaspert <joerg@debian.org>
+
+ * dak/edit_transitions.py: Done a number of cleanups to make code
+ working. Also changed the way prompting/answering goes, to not
+ have to import daklib/queue.
+ (edit_transitions): When done with a successful edit - also print
+ a final overview about defined transitions
+
+2008-03-11 Joerg Jaspert <joerg@debian.org>
+
+ * dak/process_unchecked.py: Import syck module directly, not "from
+ syck import *"
+ (check_transition): Do the check for sourceful upload in here
+ Also adjust the syck loading commands, rename new_vers to
+ expected, curvers to current, to make it more clear what they mean.
+
+ * daklib/database.py (get_suite_version): Renamed from
+ get_testing_version. Also changed the cache variables name
+
+ * The above changes are based on modifications from Anthony.
+
+ * dak/dak.py (init): Renamed check -> edit transitions
+
+ * dak/edit_transitions.py: Renamed from check_transitions.py
+ (main): Also rename new_vers/curvers to expected/current
+ Basically a nice rewrite, so it now does checks and edit,
+ depending on how you call it. Check also removes old transitions,
+ if user wants it.
+
+2008-03-02 Joerg Jaspert <joerg@debian.org>
+
+ * debian/control (Suggests): Add python-syck to Depends:
+
+ * dak/dak.py (init): Tell it about check_transitions
+
+ * dak/check_transitions.py (usage): Added, checks the transitions
+ file (if any)
+
+ * daklib/database.py (get_testing_version): Added. Returns the
+ version for the source in testing, if any
+
+ * dak/process_unchecked.py (check_transition): Added. Checks if a
+ release team member defined a transition, and rejects based on
+ that data.
+ (process_it): Use it.
+ (check_transition): Warn on broken transitions file and return,
+ not doing anything.
+ (check_transition): Moved out of here, into daklib/queue
+ (process_it): Call check_transitions only if
+ changes[architecture] has source included.
+ (check_transition): Now call the database.get_testing_version
+
2008-02-09 Christoph Berg <myon@debian.org>
+ * daklib/queue.py (get_type): fubar does not exist in global
+ namespace.
+
* setup/add_constraints.sql setup/init_pool.sql: Add changedby column
to source table, and move src_uploaders after source so the REFERNCES
clause works.
FileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/testing_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source";
+ Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source";
BinOverride "override.lenny.$(SECTION)";
ExtraOverride "override.lenny.extra.$(SECTION)";
SrcOverride "override.lenny.$(SECTION).src";
FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
SourceFileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
Sections "main contrib non-free";
- Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source";
+ Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc source";
BinOverride "override.lenny.$(SECTION)";
ExtraOverride "override.lenny.extra.$(SECTION)";
SrcOverride "override.lenny.$(SECTION).src";
{
FileList "/srv/ftp.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc";
+ Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc";
BinOverride "override.lenny.main.$(SECTION)";
SrcOverride "override.lenny.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
{
FileList "/srv/ftp.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc";
+ Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc";
BinOverride "override.lenny.main.$(SECTION)";
SrcOverride "override.lenny.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
{
FileList "/srv/ftp.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
Sections "debian-installer";
- Architectures "alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc";
+ Architectures "alpha amd64 arm armel hppa i386 ia64 mips mipsel powerpc s390 sparc";
BinOverride "override.lenny.main.$(SECTION)";
SrcOverride "override.lenny.main.src";
BinCacheDB "packages-debian-installer-$(ARCH).db";
TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
# Update fingerprints
-# [JT - disabled, dak import-ldap-fingerprints currently can ask questions]
-#dak import-ldap-fingerprints
+dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg
# Generate override files
cd $overridedir
TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
# Vacuum the database
-echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$"
+# (JJ, 20-04-2008) disabled, as we have autovacuum set to on in postgres.
+# refer to http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM
+# which says "Beginning in PostgreSQL 8.1, there is an optional feature called autovacuum,
+# whose purpose is to automate the execution of VACUUM and ANALYZE commands."
+# echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$"
+
+echo "Expiring old database dumps..."
+(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
################################################################################
################################################################################
+# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
+ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1
+
# Run mirror-split
#time dak mirror-split
TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X)
-# Compress psql backups older than a month, but no more than 20 of them
+# Compress psql backups older than a week, but no more than 20 of them
(cd $base/backup/
- find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +30 |
+ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 |
sort | head -n20 | while read dumpname; do
echo "Compressing $dumpname"
bzip2 -9 "$dumpname"
Reject
{
NoSourceOnly "true";
+ ReleaseTransitions "/srv/ftp.debian.org/testing/hints/transitions.yaml";
};
};
+Transitions
+{
+ TempPath "/srv/ftp.debian.org/tmp/";
+};
+
Binary-Upload-Restrictions
{
Components
sparc;
};
Announce "debian-changes@lists.debian.org";
- Version "3.1r6";
+ Version "3.1r8";
Origin "Debian";
- Description "Debian 3.1r6 Released 7 April 2007";
+ Description "Debian 3.1r8 Released 12 April 2008";
CodeName "sarge";
OverrideCodeName "sarge";
Priority "2";
alpha;
amd64;
arm;
+ armel;
hppa;
i386;
ia64;
alpha;
amd64;
arm;
+ armel;
hppa;
i386;
ia64;
"map oldstable-security oldstable-proposed-updates";
"map stable proposed-updates";
"map stable-security proposed-updates";
+ "map stable-proposed-updates proposed-updates";
"map-unreleased oldstable unstable";
"map-unreleased stable unstable";
"map-unreleased proposed-updates unstable";
AutomaticByHandPackages {
"debian-installer-images" {
- Source "xxx-debian-installer";
+ Source "debian-installer";
Section "raw-installer";
Extension "tar.gz";
Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-di";
-import sys
+import sys, os, textwrap
+import apt_pkg
+import daklib.utils, daklib.database
+import syck
+
+import daklib.extensions
+from daklib.extensions import replace_dak_function
+
+def check_transition():
+ changes = dak_module.changes
+ reject = dak_module.reject
+ Cnf = dak_module.Cnf
+
+ sourcepkg = changes["source"]
+
+ # No sourceful upload -> no need to do anything else, direct return
+ # We also work with unstable uploads, not experimental or those going to some
+ # proposed-updates queue
+ if "source" not in changes["architecture"] or "unstable" not in changes["distribution"]:
+ return
+
+ # Also only check if there is a file defined (and existant) with
+ # checks.
+ transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "")
+ if transpath == "" or not os.path.exists(transpath):
+ return
+
+ # Parse the yaml file
+ sourcefile = file(transpath, 'r')
+ sourcecontent = sourcefile.read()
+ try:
+ transitions = syck.load(sourcecontent)
+ except syck.error, msg:
+ # This shouldn't happen, there is a wrapper to edit the file which
+ # checks it, but we prefer to be safe than ending up rejecting
+ # everything.
+ daklib.utils.warn("Not checking transitions, the transitions file is broken: %s." % (msg))
+ return
+
+ # Now look through all defined transitions
+ for trans in transitions:
+ t = transitions[trans]
+ source = t["source"]
+ expected = t["new"]
+
+ # Will be None if nothing is in testing.
+ current = daklib.database.get_suite_version(source, "testing")
+ if current is not None:
+ compare = apt_pkg.VersionCompare(current, expected)
+
+ if current is None or compare < 0:
+ # This is still valid, the current version in testing is older than
+ # the new version we wait for, or there is none in testing yet
+
+ # Check if the source we look at is affected by this.
+ if sourcepkg in t['packages']:
+ # The source is affected, lets reject it.
+
+ rejectmsg = "%s: part of the %s transition.\n\n" % (
+ sourcepkg, trans)
+
+ if current is not None:
+ currentlymsg = "at version %s" % (current)
+ else:
+ currentlymsg = "not present in testing"
+
+ rejectmsg += "Transition description: %s\n\n" % (t["reason"])
+
+ rejectmsg += "\n".join(textwrap.wrap("""Your package
+is part of a testing transition designed to get %s migrated (it is
+currently %s, we need version %s). This transition is managed by the
+Release Team, and %s is the Release-Team member responsible for it.
+Please mail debian-release@lists.debian.org or contact %s directly if you
+need further assistance. You might want to upload to experimental until this
+transition is done."""
+ % (source, currentlymsg, expected,t["rm"], t["rm"])))
+
+ reject(rejectmsg + "\n")
+ return
+
+@replace_dak_function("process-unchecked", "check_signed_by_key")
+def check_signed_by_key(oldfn):
+ changes = dak_module.changes
+ reject = dak_module.reject
+
+ if changes["source"] == "dpkg":
+ fpr = changes["fingerprint"]
+ (uid, uid_name) = dak_module.lookup_uid_from_fingerprint(fpr)
+ if fpr == "5906F687BD03ACAD0D8E602EFCF37657" or uid == "iwj":
+ reject("Upload blocked due to hijack attempt 2008/03/19")
+
+ # NB: 1.15.0, 1.15.2 signed by this key targetted at unstable
+ # have been made available in the wild, and should remain
+ # blocked until Debian's dpkg has revved past those version
+ # numbers
+
+ oldfn()
+
+ check_transition()
#!/usr/bin/env python
# Remove obsolete .changes files from proposed-updates
-# Copyright (C) 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+# Copyright (C) 2001, 2002, 2003, 2004, 2006, 2008 James Troup <james@nocrew.org>
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
daklib.utils.fubar("unknown type, fix me")
if not pu.has_key(pkg):
# FIXME
- daklib.utils.warn("%s doesn't seem to exist in p-u?? (from %s [%s])" % (pkg, file, filename))
+ daklib.utils.warn("%s doesn't seem to exist in %s?? (from %s [%s])" % (pkg, Options["suite"], file, filename))
continue
if not pu[pkg].has_key(arch):
# FIXME
- daklib.utils.warn("%s doesn't seem to exist for %s in p-u?? (from %s [%s])" % (pkg, arch, file, filename))
+ daklib.utils.warn("%s doesn't seem to exist for %s in %s?? (from %s [%s])" % (pkg, arch, Options["suite"], file, filename))
continue
pu_version = daklib.utils.re_no_epoch.sub('', pu[pkg][arch])
if pu_version == version:
if new_num_files == 0:
print "%s: no files left, superseded by %s" % (filename, pu_version)
dest = Cnf["Dir::Morgue"] + "/misc/"
- daklib.utils.move(filename, dest)
+ if not Options["no-action"]:
+ daklib.utils.move(filename, dest)
elif new_num_files < num_files:
print "%s: lost files, MWAAP." % (filename)
else:
file = daklib.utils.open_file(filename)
cwd = os.getcwd()
- os.chdir("%s/dists/proposed-updates" % (Cnf["Dir::Root"]))
+ os.chdir("%s/dists/%s" % (Cnf["Dir::Root"]), Options["suite"])
for line in file.readlines():
line = line.rstrip()
SELECT b.package, b.version, a.arch_string
FROM bin_associations ba, binaries b, suite su, architecture a
WHERE b.id = ba.bin AND ba.suite = su.id
- AND su.suite_name = 'proposed-updates' AND a.id = b.architecture
+ AND su.suite_name = '%s' AND a.id = b.architecture
UNION SELECT s.source, s.version, 'source'
FROM src_associations sa, source s, suite su
WHERE s.id = sa.source AND sa.suite = su.id
- AND su.suite_name = 'proposed-updates'
+ AND su.suite_name = '%s'
ORDER BY package, version, arch_string
-""")
+""" % (Options["suite"], Options["suite"]))
ql = q.getresult()
for i in ql:
pkg = i[0]
Cnf = daklib.utils.get_conf()
Arguments = [('d', "debug", "Clean-Proposed-Updates::Options::Debug"),
- ('v',"verbose","Clean-Proposed-Updates::Options::Verbose"),
- ('h',"help","Clean-Proposed-Updates::Options::Help")]
- for i in [ "debug", "verbose", "help" ]:
+ ('v', "verbose", "Clean-Proposed-Updates::Options::Verbose"),
+ ('h', "help", "Clean-Proposed-Updates::Options::Help"),
+ ('s', "suite", "Clean-Proposed-Updates::Options::Suite", "HasArg"),
+ ('n', "no-action", "Clean-Proposed-Updates::Options::No-Action"),]
+ for i in [ "debug", "verbose", "help", "no-action" ]:
if not Cnf.has_key("Clean-Proposed-Updates::Options::%s" % (i)):
Cnf["Clean-Proposed-Updates::Options::%s" % (i)] = ""
+ # suite defaults to proposed-updates
+ if not Cnf.has_key("Clean-Proposed-Updates::Options::Suite"):
+ Cnf["Clean-Proposed-Updates::Options::Suite"] = "proposed-updates"
+
arguments = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
Options = Cnf.SubTree("Clean-Proposed-Updates::Options")
################################################################################
import sys, imp
-import daklib.utils
+import daklib.utils, daklib.extensions
################################################################################
"Clean cruft from incoming"),
("clean-proposed-updates",
"Remove obsolete .changes from proposed-updates"),
-
+
+ ("transitions",
+ "Manage the release transition file"),
("check-overrides",
"Override cruft checks"),
("check-proposed-updates",
module.dak_userext = userext
userext.dak_module = module
+
+ daklib.extensions.init(cmdname, module, userext)
if userext.init is not None: userext.init(cmdname)
module.main()
name = desuid_byid[id][1]
oname = db_uid_byid[id][1]
if name and oname != name:
- changes.append((uid[1], "Full name: %s\n" % (name)))
+ changes.append((uid[1], "Full name: %s" % (name)))
projectB.query("UPDATE uid SET name = '%s' WHERE id = %s" %
(pg.escape_string(name), id))
for f,(u,fid,kr) in db_fin_info.iteritems():
if kr != keyring_id: continue
if f in fpr: continue
- changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s\n" % (f)))
+ changes.append((db_uid_byid.get(u, [None])[0], "Removed key: %s" % (f)))
projectB.query("UPDATE fingerprint SET keyring = NULL WHERE id = %d" % (fid))
# For the keys in this keyring, add/update any fingerprints that've
if olduid == None: olduid = -1
if oldkid == None: oldkid = -1
if oldfid == -1:
- changes.append((newuiduid, "Added key: %s\n" % (f)))
+ changes.append((newuiduid, "Added key: %s" % (f)))
if newuid:
projectB.query("INSERT INTO fingerprint (fingerprint, uid, keyring) VALUES ('%s', %d, %d)" % (f, newuid, keyring_id))
else:
else:
if newuid and olduid != newuid:
if olduid != -1:
- changes.append((newuiduid, "Linked key: %s (formerly belonging to %s)" % (f, db_uid_byid[olduid][0])))
+ changes.append((newuiduid, "Linked key: %s" % f))
+ changes.append((newuiduid, " (formerly belonging to %s)" % (db_uid_byid[olduid][0])))
else:
- changes.append((newuiduid, "Linked key: %s (formerly unowned)\n" % (f)))
+ changes.append((newuiduid, "Linked key: %s" % f))
+ changes.append((newuiduid, " (formerly unowned)"))
projectB.query("UPDATE fingerprint SET uid = %d WHERE id = %d" % (newuid, oldfid))
if oldkid != keyring_id:
changesd = {}
for (k, v) in changes:
if k not in changesd: changesd[k] = ""
- changesd[k] += " " + v
+ changesd[k] += " %s\n" % (v)
keys = changesd.keys()
keys.sort()
for k in keys:
- print "%s\n%s" % (k, changesd[k])
+ print "%s\n%s\n" % (k, changesd[k])
################################################################################
if dsc.get("dm-upload-allowed", "no") == "yes":
uploader_ids = [maintainer_id]
if dsc.has_key("uploaders"):
- for u in dsc["uploaders"].split(","):
- u = u.replace("'", "\\'")
- u = u.strip()
+ for u in dsc["uploaders"].split(","):
+ u = u.replace("'", "\\'")
+ u = u.strip()
uploader_ids.append(
- daklib.database.get_or_set_maintainer_id(u))
+ daklib.database.get_or_set_maintainer_id(u))
+ added_ids = {}
for u in uploader_ids:
+ if added_ids.has_key(u):
+ daklib.utils.warn("Already saw uploader %s for source %s" % (u, package))
+ continue
+ added_ids[u]=1
projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u))
################################################################################
-def check_md5sums ():
- for file in files.keys():
+def check_hashes ():
+ # Make sure we recognise the format of the Files: field
+ format = changes.get("format", "0.0").split(".",1)
+ if len(format) == 2:
+ format = int(format[0]), int(format[1])
+ else:
+ format = int(float(format[0])), 0
+
+ check_hash(".changes", files, "md5sum", apt_pkg.md5sum)
+ check_hash(".dsc", dsc_files, "md5sum", apt_pkg.md5sum)
+
+ if format >= (1,8):
+ hashes = [("sha1", apt_pkg.sha1sum),
+ ("sha256", apt_pkg.sha256sum)]
+ else:
+ hashes = []
+
+ for x in changes:
+ if x.startswith("checksum-"):
+ h = x.split("-",1)[1]
+ if h not in dict(hashes):
+ reject("Unsupported checksum field in .changes" % (h))
+
+ for x in dsc:
+ if x.startswith("checksum-"):
+ h = x.split("-",1)[1]
+ if h not in dict(hashes):
+ reject("Unsupported checksum field in .dsc" % (h))
+
+ for h,f in hashes:
try:
- file_handle = daklib.utils.open_file(file)
- except daklib.utils.cant_open_exc:
- continue
+ fs = daklib.utils.build_file_list(changes, 0, "checksums-%s" % h, h)
+ check_hash(".changes %s" % (h), fs, h, f, files)
+ except daklib.utils.no_files_exc:
+ reject("No Checksums-%s: field in .changes file" % (h))
- # Check md5sum
- if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
- reject("%s: md5sum check failed." % (file))
- file_handle.close()
- # Check size
- actual_size = os.stat(file)[stat.ST_SIZE]
- size = int(files[file]["size"])
- if size != actual_size:
- reject("%s: actual file size (%s) does not match size (%s) in .changes"
- % (file, actual_size, size))
+ if "source" not in changes["architecture"]: continue
+
+ try:
+ fs = daklib.utils.build_file_list(dsc, 1, "checksums-%s" % h, h)
+ check_hash(".dsc %s" % (h), fs, h, f, dsc_files)
+ except daklib.utils.no_files_exc:
+ reject("No Checksums-%s: field in .changes file" % (h))
+
+################################################################################
+
+def check_hash (where, files, key, testfn, basedict = None):
+ if basedict:
+ for file in basedict.keys():
+ if file not in files:
+ reject("%s: no %s checksum" % (file, key))
+
+ for file in files.keys():
+ if basedict and file not in basedict:
+ reject("%s: extraneous entry in %s checksums" % (file, key))
- for file in dsc_files.keys():
try:
file_handle = daklib.utils.open_file(file)
except daklib.utils.cant_open_exc:
continue
- # Check md5sum
- if apt_pkg.md5sum(file_handle) != dsc_files[file]["md5sum"]:
- reject("%s: md5sum check failed." % (file))
+ # Check hash
+ if testfn(file_handle) != files[file][key]:
+ reject("%s: %s check failed." % (file, key))
file_handle.close()
# Check size
actual_size = os.stat(file)[stat.ST_SIZE]
- size = int(dsc_files[file]["size"])
+ size = int(files[file]["size"])
if size != actual_size:
- reject("%s: actual file size (%s) does not match size (%s) in .dsc"
- % (file, actual_size, size))
+ reject("%s: actual file size (%s) does not match size (%s) in %s"
+ % (file, actual_size, size, where))
################################################################################
valid_dsc_p = check_dsc()
if valid_dsc_p:
check_source()
- check_md5sums()
+ check_hashes()
check_urgency()
check_timestamps()
check_signed_by_key()
--- /dev/null
+#!/usr/bin/env python
+
+# Display, edit and check the release manager's transition file.
+# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <elmo> if klecker.d.o died, I swear to god, I'm going to migrate to gentoo.
+
+################################################################################
+
+import os, pg, sys, time, errno, fcntl, tempfile, pwd, re
+import apt_pkg
+import daklib.database
+import daklib.utils
+import syck
+
+# Globals
+Cnf = None
+Options = None
+projectB = None
+
+re_broken_package = re.compile(r"[a-zA-Z]\w+\s+\-.*")
+
+################################################################################
+
+#####################################
+#### This may run within sudo !! ####
+#####################################
+def init():
+ global Cnf, Options, projectB
+
+ apt_pkg.init()
+
+ Cnf = daklib.utils.get_conf()
+
+ Arguments = [('h',"help","Edit-Transitions::Options::Help"),
+ ('e',"edit","Edit-Transitions::Options::Edit"),
+ ('i',"import","Edit-Transitions::Options::Import", "HasArg"),
+ ('c',"check","Edit-Transitions::Options::Check"),
+ ('s',"sudo","Edit-Transitions::Options::Sudo"),
+ ('n',"no-action","Edit-Transitions::Options::No-Action")]
+
+ for i in ["help", "no-action", "edit", "import", "check", "sudo"]:
+ if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)):
+ Cnf["Edit-Transitions::Options::%s" % (i)] = ""
+
+ apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
+
+ Options = Cnf.SubTree("Edit-Transitions::Options")
+
+ if Options["help"]:
+ usage()
+
+ whoami = os.getuid()
+ whoamifull = pwd.getpwuid(whoami)
+ username = whoamifull[0]
+ if username != "dak":
+ print "Non-dak user: %s" % username
+ Options["sudo"] = "y"
+
+ projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]))
+ daklib.database.init(Cnf, projectB)
+
+################################################################################
+
+def usage (exit_code=0):
+ print """Usage: transitions [OPTION]...
+Update and check the release managers transition file.
+
+Options:
+
+ -h, --help show this help and exit.
+ -e, --edit edit the transitions file
+ -i, --import <file> check and import transitions from file
+ -c, --check check the transitions file, remove outdated entries
+ -S, --sudo use sudo to update transitions file
+ -n, --no-action don't do anything (only affects check)"""
+
+ sys.exit(exit_code)
+
+################################################################################
+
+#####################################
+#### This may run within sudo !! ####
+#####################################
+def load_transitions(trans_file):
+ # Parse the yaml file
+ sourcefile = file(trans_file, 'r')
+ sourcecontent = sourcefile.read()
+ failure = False
+ try:
+ trans = syck.load(sourcecontent)
+ except syck.error, msg:
+ # Someone fucked it up
+ print "ERROR: %s" % (msg)
+ return None
+
+ # lets do further validation here
+ checkkeys = ["source", "reason", "packages", "new", "rm"]
+
+ # If we get an empty definition - we just have nothing to check, no transitions defined
+ if type(trans) != dict:
+ # This can be anything. We could have no transitions defined. Or someone totally fucked up the
+ # file, adding stuff in a way we dont know or want. Then we set it empty - and simply have no
+ # transitions anymore. User will see it in the information display after he quit the editor and
+ # could fix it
+ trans = ""
+ return trans
+
+ try:
+ for test in trans:
+ t = trans[test]
+
+ # First check if we know all the keys for the transition and if they have
+ # the right type (and for the packages also if the list has the right types
+ # included, ie. not a list in list, but only str in the list)
+ for key in t:
+ if key not in checkkeys:
+ print "ERROR: Unknown key %s in transition %s" % (key, test)
+ failure = True
+
+ if key == "packages":
+ if type(t[key]) != list:
+ print "ERROR: Unknown type %s for packages in transition %s." % (type(t[key]), test)
+ failure = True
+ try:
+ for package in t["packages"]:
+ if type(package) != str:
+ print "ERROR: Packages list contains invalid type %s (as %s) in transition %s" % (type(package), package, test)
+ failure = True
+ if re_broken_package.match(package):
+ # Someone had a space too much (or not enough), we have something looking like
+ # "package1 - package2" now.
+ print "ERROR: Invalid indentation of package list in transition %s, around package(s): %s" % (test, package)
+ failure = True
+ except TypeError:
+ # In case someone has an empty packages list
+ print "ERROR: No packages defined in transition %s" % (test)
+ failure = True
+ continue
+
+ elif type(t[key]) != str:
+ if key == "new" and type(t[key]) == int:
+ # Ok, debian native version
+ continue
+ else:
+ print "ERROR: Unknown type %s for key %s in transition %s" % (type(t[key]), key, test)
+ failure = True
+
+ # And now the other way round - are all our keys defined?
+ for key in checkkeys:
+ if key not in t:
+ print "ERROR: Missing key %s in transition %s" % (key, test)
+ failure = True
+ except TypeError:
+ # In case someone defined very broken things
+ print "ERROR: Unable to parse the file"
+ failure = True
+
+
+ if failure:
+ return None
+
+ return trans
+
+################################################################################
+
+#####################################
+#### This may run within sudo !! ####
+#####################################
+def lock_file(file):
+ for retry in range(10):
+ lock_fd = os.open(file, os.O_RDWR | os.O_CREAT)
+ try:
+ fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ return lock_fd
+ except OSError, e:
+ if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
+ print "Unable to get lock for %s (try %d of 10)" % \
+ (file, retry+1)
+ time.sleep(60)
+ else:
+ raise
+
+ daklib.utils.fubar("Couldn't obtain lock for %s." % (lockfile))
+
+################################################################################
+
+#####################################
+#### This may run within sudo !! ####
+#####################################
+def write_transitions(from_trans):
+ """Update the active transitions file safely.
+ This function takes a parsed input file (which avoids invalid
+ files or files that may be be modified while the function is
+ active), and ensure the transitions file is updated atomically
+ to avoid locks."""
+
+ trans_file = Cnf["Dinstall::Reject::ReleaseTransitions"]
+ trans_temp = trans_file + ".tmp"
+
+ trans_lock = lock_file(trans_file)
+ temp_lock = lock_file(trans_temp)
+
+ destfile = file(trans_temp, 'w')
+ syck.dump(from_trans, destfile)
+ destfile.close()
+
+ os.rename(trans_temp, trans_file)
+ os.close(temp_lock)
+ os.close(trans_lock)
+
+################################################################################
+
+class ParseException(Exception):
+ pass
+
+##########################################
+#### This usually runs within sudo !! ####
+##########################################
+def write_transitions_from_file(from_file):
+ """We have a file we think is valid; if we're using sudo, we invoke it
+ here, otherwise we just parse the file and call write_transitions"""
+
+ # Lets check if from_file is in the directory we expect it to be in
+ if not os.path.abspath(from_file).startswith(Cnf["Transitions::TempPath"]):
+ print "Will not accept transitions file outside of %s" % (Cnf["Transitions::TempPath"])
+ sys.exit(3)
+
+ if Options["sudo"]:
+ os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H",
+ "/usr/local/bin/dak", "transitions", "--import", from_file)
+ else:
+ trans = load_transitions(from_file)
+ if trans is None:
+ raise ParseException, "Unparsable transitions file %s" % (file)
+ write_transitions(trans)
+
+################################################################################
+
+def temp_transitions_file(transitions):
+ # NB: file is unlinked by caller, but fd is never actually closed.
+ # We need the chmod, as the file is (most possibly) copied from a
+ # sudo-ed script and would be unreadable if it has default mkstemp mode
+
+ (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Transitions::TempPath"])
+ os.chmod(path, 0644)
+ f = open(path, "w")
+ syck.dump(transitions, f)
+ return path
+
+################################################################################
+
+def edit_transitions():
+ trans_file = Cnf["Dinstall::Reject::ReleaseTransitions"]
+ edit_file = temp_transitions_file(load_transitions(trans_file))
+
+ editor = os.environ.get("EDITOR", "vi")
+
+ while True:
+ result = os.system("%s %s" % (editor, edit_file))
+ if result != 0:
+ os.unlink(edit_file)
+ daklib.utils.fubar("%s invocation failed for %s, not removing tempfile." % (editor, edit_file))
+
+ # Now try to load the new file
+ test = load_transitions(edit_file)
+
+ if test == None:
+ # Edit is broken
+ print "Edit was unparsable."
+ prompt = "[E]dit again, Drop changes?"
+ default = "E"
+ else:
+ print "Edit looks okay.\n"
+ print "The following transitions are defined:"
+ print "------------------------------------------------------------------------"
+ transition_info(test)
+
+ prompt = "[S]ave, Edit again, Drop changes?"
+ default = "S"
+
+ answer = "XXX"
+ while prompt.find(answer) == -1:
+ answer = daklib.utils.our_raw_input(prompt)
+ if answer == "":
+ answer = default
+ answer = answer[:1].upper()
+
+ if answer == 'E':
+ continue
+ elif answer == 'D':
+ os.unlink(edit_file)
+ print "OK, discarding changes"
+ sys.exit(0)
+ elif answer == 'S':
+ # Ready to save
+ break
+ else:
+ print "You pressed something you shouldn't have :("
+ sys.exit(1)
+
+ # We seem to be done and also have a working file. Copy over.
+ write_transitions_from_file(edit_file)
+ os.unlink(edit_file)
+
+ print "Transitions file updated."
+
+################################################################################
+
+def check_transitions(transitions):
+ to_dump = 0
+ to_remove = []
+ # Now look through all defined transitions
+ for trans in transitions:
+ t = transitions[trans]
+ source = t["source"]
+ expected = t["new"]
+
+ # Will be None if nothing is in testing.
+ current = daklib.database.get_suite_version(source, "testing")
+
+ print_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
+
+ if current == None:
+ # No package in testing
+ print "Transition source %s not in testing, transition still ongoing." % (source)
+ else:
+ compare = apt_pkg.VersionCompare(current, expected)
+ if compare < 0:
+ # This is still valid, the current version in database is older than
+ # the new version we wait for
+ print "This transition is still ongoing, we currently have version %s" % (current)
+ else:
+ print "REMOVE: This transition is over, the target package reached testing. REMOVE"
+ print "%s wanted version: %s, has %s" % (source, expected, current)
+ to_remove.append(trans)
+ to_dump = 1
+ print "-------------------------------------------------------------------------"
+
+ if to_dump:
+ prompt = "Removing: "
+ for remove in to_remove:
+ prompt += remove
+ prompt += ","
+
+ prompt += " Commit Changes? (y/N)"
+ answer = ""
+
+ if Options["no-action"]:
+ answer="n"
+ else:
+ answer = daklib.utils.our_raw_input(prompt).lower()
+
+ if answer == "":
+ answer = "n"
+
+ if answer == 'n':
+ print "Not committing changes"
+ sys.exit(0)
+ elif answer == 'y':
+ print "Committing"
+ for remove in to_remove:
+ del transitions[remove]
+
+ edit_file = temp_transitions_file(transitions)
+ write_transitions_from_file(edit_file)
+
+ print "Done"
+ else:
+ print "WTF are you typing?"
+ sys.exit(0)
+
+################################################################################
+
+def print_info(trans, source, expected, rm, reason, packages):
+ print """Looking at transition: %s
+ Source: %s
+ New Version: %s
+ Responsible: %s
+ Description: %s
+ Blocked Packages (total: %d): %s
+""" % (trans, source, expected, rm, reason, len(packages), ", ".join(packages))
+ return
+
+################################################################################
+
+def transition_info(transitions):
+ for trans in transitions:
+ t = transitions[trans]
+ source = t["source"]
+ expected = t["new"]
+
+ # Will be None if nothing is in testing.
+ current = daklib.database.get_suite_version(source, "testing")
+
+ print_info(trans, source, expected, t["rm"], t["reason"], t["packages"])
+
+ if current == None:
+ # No package in testing
+ print "Transition source %s not in testing, transition still ongoing." % (source)
+ else:
+ compare = apt_pkg.VersionCompare(current, expected)
+ print "Apt compare says: %s" % (compare)
+ if compare < 0:
+ # This is still valid, the current version in database is older than
+ # the new version we wait for
+ print "This transition is still ongoing, we currently have version %s" % (current)
+ else:
+ print "This transition is over, the target package reached testing, should be removed"
+ print "%s wanted version: %s, has %s" % (source, expected, current)
+ print "-------------------------------------------------------------------------"
+
+################################################################################
+
+def main():
+ global Cnf
+
+ #####################################
+ #### This can run within sudo !! ####
+ #####################################
+ init()
+
+ # Check if there is a file defined (and existant)
+ transpath = Cnf.get("Dinstall::Reject::ReleaseTransitions", "")
+ if transpath == "":
+ daklib.utils.warn("Dinstall::Reject::ReleaseTransitions not defined")
+ sys.exit(1)
+ if not os.path.exists(transpath):
+ daklib.utils.warn("ReleaseTransitions file, %s, not found." %
+ (Cnf["Dinstall::Reject::ReleaseTransitions"]))
+ sys.exit(1)
+ # Also check if our temp directory is defined and existant
+ temppath = Cnf.get("Transitions::TempPath", "")
+ if temppath == "":
+ daklib.utils.warn("Transitions::TempPath not defined")
+ sys.exit(1)
+ if not os.path.exists(temppath):
+ daklib.utils.warn("Temporary path %s not found." %
+ (Cnf["Transitions::TempPath"]))
+ sys.exit(1)
+
+ if Options["import"]:
+ try:
+ write_transitions_from_file(Options["import"])
+ except ParseException, m:
+ print m
+ sys.exit(2)
+ sys.exit(0)
+ ##############################################
+ #### Up to here it can run within sudo !! ####
+ ##############################################
+
+ # Parse the yaml file
+ transitions = load_transitions(transpath)
+ if transitions == None:
+ # Something very broken with the transitions, exit
+ daklib.utils.warn("Could not parse existing transitions file. Aborting.")
+ sys.exit(2)
+
+ if Options["edit"]:
+ # Let's edit the transitions file
+ edit_transitions()
+ elif Options["check"]:
+ # Check and remove outdated transitions
+ check_transitions(transitions)
+ else:
+ # Output information about the currently defined transitions.
+ print "Currently defined transitions:"
+ transition_info(transitions)
+
+ sys.exit(0)
+
+################################################################################
+
+if __name__ == '__main__':
+ main()
fingerprint_id_cache = {}
queue_id_cache = {}
uid_id_cache = {}
+suite_version_cache = {}
################################################################################
return source_id
+def get_suite_version(source, suite):
+ global suite_version_cache
+ cache_key = "%s_%s" % (source, suite)
+
+ if suite_version_cache.has_key(cache_key):
+ return suite_version_cache[cache_key]
+
+ q = projectB.query("""
+ SELECT s.version FROM source s, suite su, src_associations sa
+ WHERE sa.source=s.id
+ AND sa.suite=su.id
+ AND su.suite_name='%s'
+ AND s.source='%s'"""
+ % (suite, source))
+
+ if not q.getresult():
+ return None
+
+ version = q.getresult()[0][0]
+ suite_version_cache[cache_key] = version
+
+ return version
+
################################################################################
def get_or_set_maintainer_id (maintainer):
--- /dev/null
+#!/usr/bin/env python
+
+# Utility functions for extensions
+# Copyright (C) 2008 Anthony Towns <ajt@dbeian.org>
+
+################################################################################
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+dak_functions_to_replace = {}
+dak_replaced_functions = {}
+
+def replace_dak_function(module,name):
+ """Decorator to make a function replace a standard dak function
+ in a given module. The replaced function will be provided as
+ the first argument."""
+
+ def x(f):
+ def myfunc(*a,**kw):
+ global replaced_funcs
+ f(dak_replaced_functions[name], *a, **kw)
+ myfunc.__name__ = f.__name__
+ myfunc.__doc__ = f.__doc__
+ myfunc.__dict__.update(f.__dict__)
+
+ fnname = "%s:%s" % (module, name)
+ if fnname in dak_functions_to_replace:
+ raise Exception, \
+ "%s in %s already marked to be replaced" % (name, module)
+ dak_functions_to_replace["%s:%s" % (module,name)] = myfunc
+ return f
+ return x
+
+################################################################################
+
+def init(name, module, userext):
+ global dak_replaced_functions
+
+ # This bit should be done automatically too
+ dak_replaced_functions = {}
+ for f,newfunc in dak_functions_to_replace.iteritems():
+ m,f = f.split(":",1)
+ if len(f) > 0 and m == name:
+ dak_replaced_functions[f] = module.__dict__[f]
+ module.__dict__[f] = newfunc
+
+
elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
type = "dsc"
else:
- fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
+ utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
# Validate the override type
type_id = database.get_override_type_id(type)
if type_id == -1:
- fubar("invalid type (%s) for new. Say wha?" % (type))
+ utils.fubar("invalid type (%s) for new. Say wha?" % (type))
return type
re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")
re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$")
+re_verwithext = re.compile(r"^(\d+)(?:\.(\d+))(?:\s+\((\S+)\))?$")
changes_parse_error_exc = "Can't parse line in .changes file"
invalid_dsc_format_exc = "Invalid .dsc file"
# Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
-def build_file_list(changes, is_a_dsc=0):
+def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
files = {}
# Make sure we have a Files: field to parse...
- if not changes.has_key("files"):
- raise no_files_exc
+ if not changes.has_key(field):
+ raise no_files_exc
# Make sure we recognise the format of the Files: field
- format = changes.get("format", "")
- if format != "":
- format = float(format)
- if not is_a_dsc and (format < 1.5 or format > 2.0):
- raise nk_format_exc, format
+ format = re_verwithext.search(changes.get("format", "0.0"))
+ if not format:
+ raise nk_format_exc, "%s" % (changes.get("format","0.0"))
+
+ format = format.groups()
+ if format[1] == None:
+ format = int(float(format[0])), 0, format[2]
+ else:
+ format = int(format[0]), int(format[1]), format[2]
+ if format[2] == None:
+ format = format[:2]
+
+ if is_a_dsc:
+ if format != (1,0):
+ raise nk_format_exc, "%s" % (changes.get("format","0.0"))
+ else:
+ if (format < (1,5) or format > (1,8)):
+ raise nk_format_exc, "%s" % (changes.get("format","0.0"))
+ if field != "files" and format < (1,8):
+ raise nk_format_exc, "%s" % (changes.get("format","0.0"))
+
+ includes_section = (not is_a_dsc) and field == "files"
# Parse each entry/line:
- for i in changes["files"].split('\n'):
+ for i in changes[field].split('\n'):
if not i:
break
s = i.split()
section = priority = ""
try:
- if is_a_dsc:
- (md5, size, name) = s
- else:
+ if includes_section:
(md5, size, section, priority, name) = s
+ else:
+ (md5, size, name) = s
except ValueError:
raise changes_parse_error_exc, i
(section, component) = extract_component_from_section(section)
- files[name] = Dict(md5sum=md5, size=size, section=section,
+ files[name] = Dict(size=size, section=section,
priority=priority, component=component)
+ files[name][hashname] = md5
return files
Package: dak
Architecture: any
-Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev
+Depends: ${python:Depends}, python-pygresql, python2.1-email | python (>= 2.2), python-apt, apt-utils, gnupg (>= 1.0.6-1), ${shlibs:Depends}, dpkg-dev, python-syck (>= 0.61.2-1)
Suggests: lintian, linda, less, binutils-multiarch, symlinks, postgresql (>= 7.1.0), dsync
Description: Debian's archive maintenance scripts
This is a collection of archive maintenance scripts used by the
NB: removing packages are not logged to the stable ChangeLog; you
need to do that byhand.
-o Do anything in proposed-updates/TODO
+o If you installed a debian-installer upload; migrate the relevant
+ installer-*/$release directory from proposed-updates to stable.
+ (Including potentially removing older versions)
+
+o Decruft stable in coordination with SRMs
+
+o Do anything in proposed-updates/TODO
o Close any applicable stable bugs
(hint: http://bugs.debian.org/cgi-bin/pkgreport.cgi?pkg=ftp.debian.org&include=etch)
-o Update version number in README, README.html and dists/README (ftp-master only)
+o Update version number in README, README.html and dists/README
o Update the 'Debian<n>.<n>r<n>' symlink in dists/
o Clean up dists/stable/ChangeLog (add header, basically)
o Update version fields in dak.conf
o Update fields in suite table in postgresql (see below)
-o Comment out "Untouchable" in dak.conf
-o Run 'dak make-suite-file-list -s stable'
+o Run 'dak make-suite-file-list --force -s stable'
o Run apt-ftparchive generate apt.conf.stable
-o Run 'dak generate-releases stable' ** FIXME: requires apt.conf.stable stanza for stable in apt.conf
- ** FIXME: must be run as dak
-o Uncomment "Untouchable" in dak.conf
+o Run 'dak generate-releases --force-touch --apt-conf apt.conf.stable stable'
-Yes, this sucks and more of it should be automated.
+[Yes, this sucks and more of it should be automated. c.f. ~ajt/pointupdate]
#######################################################
-update suite set version = '4.0r1' where suite_name = 'stable';
-update suite set description = 'Debian 4.0r1 Released 15th August 2007' where suite_name = 'stable';
+update suite set version = '4.0r3' where suite_name = 'stable';
+update suite set description = 'Debian 4.0r3 Released 16th February 2008' where suite_name = 'stable';
+
+Rough Guide to doing Old-Stable Point Releases in Debian
+--------------------------------------------------------
+
+Pretty much as above, except that process-accepted doesn't know about
+oldstable, so you have to do some surgery on it first to make it
+support that. Probably want to disable cron.daily whilst doing so.
+Also watch out for the installing_to_stable detection which doesn't
+work well with the current layout of oldstable-proposed-updates (as a
+symlink to $distro-proposed-updates). clean-proposed-updates,
+cruft-report and most everything else support a -s/--suite so they
+sould be fine to use.
--- /dev/null
+Contents:
+
+1. Little "Howto Use it"
+2. Explanation of how it works
+
+
+1. Little "Howto Use it"
+------------------------
+
+The input file is in YAML format. Do bnot bother with comments, they
+will be removed.
+
+The format: Dont use tabs for indentation, use spaces.
+
+Strings should be within "", but normally work without.
+Exception: Version-numbers with an epoch really do want to be in
+"". YES, THEY WANT TO (or they get interpreted in a way you dont expect
+it).
+
+Keys (The order of them does not matter, only the indentation):
+
+short_tag: A short tag for the transition, like apt_update
+ reason: One-line reason what is intended with it
+ source: Source package that needs to transition
+ new: New version of the target package
+ rm: Name of the Release Team member responsible for this transition
+ packages: Array of package names that are affected by this transition
+
+
+The following example wants to
+a.) update apt to version 0.7.12, the responsible Release Team member
+is Andreas Barth, and it affects some apt related packages and
+b.) wants to do something similar for lintian.
+
+apt_update:
+ packages:
+ - apt
+ - synaptic
+ - cron-apt
+ - debtags
+ - feta
+ - apticron
+ - aptitude
+ reason: "Apt needs to transition to testing to get foo and bar done"
+ source: apt
+ new: 0.7.12
+ rm: Andreas Barth
+lintian_breakage:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.23.45~bpo40+1
+ rm: Ganneff
+ packages:
+ - lintian
+ - python-syck
+
+
+########################################################################
+########################################################################
+
+
+2. Explanation of how it works
+------------------------------
+
+Assume the following transition is defined:
+
+lintian_funtest:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.23.45~bpo40+1
+ rm: Ganneff
+ packages:
+ - lintian
+ - python-syck
+
+Also assume the lintian situation on this archive looks like this:
+ lintian | 1.23.28~bpo.1 | sarge-backports | source, all
+ lintian | 1.23.45~bpo40+1 | etch-backports | source, all
+
+------------------------------------------------------------------------
+
+Now, I try to upload a (NEW, but that makes no difference) version of
+python-syck:
+
+$ dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes
+
+python-syck_0.61.2-1~bpo40+1_i386.changes
+REJECT
+Rejected: python-syck: part of the lintian_funtest transition.
+
+Your package is part of a testing transition designed to get lintian migrated
+(it currently is at version 1.23.28~bpo.1, we need version 1.23.45~bpo40+1)
+
+Transition description: Testing a new feature
+
+This transition is managed by the Release Team, and Ganneff
+is the Release-Team member responsible for it.
+Please contact Ganneff or debian-release@lists.debian.org if you
+need further assistance.
+
+------------------------------------------------------------------------
+
+Lets change the definition of the transition, assume it is now:
+
+lintian_funtest:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.22.28~bpo.1
+ rm: Ganneff
+ packages:
+ - lintian
+ - python-syck
+
+Which checks for a version older than the version actually available. Result:
+
+dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes
+
+python-syck_0.61.2-1~bpo40+1_i386.changes
+NEW for etch-backports
+(new) python-syck_0.61.2-1~bpo40+1.diff.gz extra python
+(new) python-syck_0.61.2-1~bpo40+1.dsc extra python
+(new) python-syck_0.61.2-1~bpo40+1_i386.deb extra python
+PySyck python bindings to the Syck YAML parser kit
+ Syck is a simple YAML parser kit.
+ .
+[...] the whole stuff about a new package.
+
+------------------------------------------------------------------------
+
+For completness, change the transition to (exact existing version):
+lintian_funtest:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.23.28~bpo.1
+ rm: Ganneff
+ packages:
+ - lintian
+
+and the result is:
+
+dak process-unchecked -n python-syck_0.61.2-1~bpo40+1_i386.changes
+
+python-syck_0.61.2-1~bpo40+1_i386.changes
+NEW for etch-backports
+[... we know this ...]
+
+------------------------------------------------------------------------
+
+The second part is the check_transitions script.
+For that we take the following transitions as example:
+
+apt_update:
+ reason: "Apt needs to transition to testing to get foo and bar done"
+ source: apt
+ new: 0.2.12-1+b1.3
+ rm: Andreas Barth
+ packages:
+ - apt
+ - synaptic
+ - cron-apt
+ - debtags
+ - feta
+ - apticron
+ - aptitude
+lintian_funtest:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.23.45~bpo40+1
+ rm: Ganneff
+ packages:
+ - lintian
+ - python-syck
+bar_breaks_it:
+ reason: We dont want bar to break it
+ source: bar
+ new: "9:99"
+ rm: Ganneff
+ packages:
+ - kdelibs
+ - qt4-x11
+ - libqt-perl
+
+Running check-transitions ends up with the following output:
+
+Looking at transition: lintian_funtest
+ Source: lintian
+ New Version: 1.23.45~bpo40+1
+ Responsible: Ganneff
+ Description: Testing a new feature
+ Blocked Packages (total: 2): lintian, python-syck
+
+Apt compare says: -2
+This transition is still ongoing, we currently have version 1.23.28~bpo.1
+-------------------------------------------------------------------------
+
+Looking at transition: apt_update
+ Source: apt
+ New Version: 0.2.12-1+b1.3
+ Responsible: Andreas Barth
+ Description: Apt needs to transition to testing to get foo and bar done
+ Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude
+
+Apt compare says: 4
+This transition is over, the target package reached testing, removing
+apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1
+-------------------------------------------------------------------------
+
+Looking at transition: bar_breaks_it
+ Source: bar
+ New Version: 9:99
+ Responsible: Ganneff
+ Description: We dont want bar to break it
+ Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl
+
+Transition source bar not in testing, transition still ongoing.
+-------------------------------------------------------------------------
+I: I would remove the apt_update transition
+
+
+Changing our transition definitions for lintian (keeping the rest as
+above) to
+
+lintian_funtest:
+ reason: "Testing a new feature"
+ source: lintian
+ new: 1.22.28~bpo.1
+ rm: Ganneff
+ packages:
+ - lintian
+ - python-syck
+
+now we get
+
+Looking at transition: lintian_funtest
+ Source: lintian
+ New Version: 1.22.28~bpo.1
+ Responsible: Ganneff
+ Description: Testing a new feature
+ Blocked Packages (total: 2): lintian, python-syck
+
+Apt compare says: 1
+This transition is over, the target package reached testing, removing
+lintian wanted version: 1.22.28~bpo.1, has 1.23.28~bpo.1
+-------------------------------------------------------------------------
+
+Looking at transition: apt_update
+ Source: apt
+ New Version: 0.2.12-1+b1.3
+ Responsible: Andreas Barth
+ Description: Apt needs to transition to testing to get foo and bar done
+ Blocked Packages (total: 7): apt, synaptic, cron-apt, debtags, feta, apticron, aptitude
+
+Apt compare says: 4
+This transition is over, the target package reached testing, removing
+apt wanted version: 0.2.12-1+b1.3, has 0.6.46.4-0.1~bpo.1
+-------------------------------------------------------------------------
+
+Looking at transition: bar_breaks_it
+ Source: bar
+ New Version: 9:99
+ Responsible: Ganneff
+ Description: We dont want bar to break it
+ Blocked Packages (total: 3): kdelibs, qt4-x11, libqt-perl
+
+Transition source bar not in testing, transition still ongoing.
+-------------------------------------------------------------------------
+I: I would remove the lintian_funtest transition
+I: I would remove the apt_update transition
+
+
+Not using the -n switch would turn the I: in actual removals :)
+The check-transition command is meant for the release team to always run
+it when they change a transition definition. It checks if the yaml is
+valid and can be loaded (but if not the archive simply does no reject)
+and also shows a nice overview.
--- /dev/null
+#!/bin/sh -ue
+
+if [ $# -lt 4 ]; then
+ echo "Usage: $0 filename version arch changes_file"
+ exit 1
+fi
+
+TARBALL="$1" # Tarball to read, compressed with gzip
+VERSION="$2"
+ARCH="$3"
+CHANGES="$4" # Changes file for the upload
+
+error() {
+ echo "$*"
+ exit 1
+}
+
+# Check validity of version number
+# Expected are: YYYYMMDD, YYYYMMDD.x, YYYYMMDD<suite>x
+if ! echo "$VERSION" | grep -Eq "^[0-9]{8}(|(\.|[a-z]+)[0-9]+)$"; then
+ error "Invalid version: '$VERSION'"
+fi
+
+# Get the target suite from the Changes file
+# NOTE: it may be better to pass this to the script as a parameter!
+SUITE="$(grep "^Distribution:" "$CHANGES" | awk '{print $2}')"
+case $SUITE in
+ "")
+ error "Error: unable to determine suite from Changes file"
+ ;;
+ unstable|sid)
+ : # nothing to do
+ ;;
+ *)
+ SUITE="${SUITE}-proposed-updates"
+ ;;
+esac
+
+# This must end with /
+TARGET="/srv/ftp.debian.org/ftp/dists/$SUITE/main/installer-$ARCH/"
+
+# Check validity of the target directory
+# This could fail, for example for new architectures; doing
+# a regular BYHAND is safer in that case
+if [ ! -d "$TARGET" ]; then
+ mkdir -p "$TARGET"
+fi
+# Check that there isn't already a directory for this version
+if [ -d "$TARGET/$VERSION" ]; then
+ error "Directory already exists: $TARGET/$VERSION"
+fi
+
+# We know all data to be in ./installer-<arch>/<version>; see if there's
+# anything else in the tarball except that and the 'current' symlink
+if tar tzf "$TARBALL" | \
+ grep -Eqv "^\./(installer-$ARCH/($VERSION/.*|current|)|)$"; then
+ error "Tarball contains unexpected contents"
+fi
+
+# Create a temporary directory where to store the images
+umask 002
+TMPDIR="$(mktemp -td byhand-di.XXXXXX)"
+
+# If we fail somewhere, cleanup the temporary directory
+cleanup() {
+ rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+# Extract the data into the temporary directory
+tar xzf "$TARBALL" --directory="$TMPDIR" "./installer-$ARCH/"
+
+# Check the 'current' symlink
+if [ ! -L $TMPDIR/installer-$ARCH/current ]; then
+ error "Missing 'current' symlink"
+elif [ X"$(readlink "$TMPDIR/installer-$ARCH/current")" != X"$VERSION" ]; then
+ error "Incorrect 'current' symlink"
+fi
+
+# We should have an MD5SUMS file; use that for a final check
+if [ -r "$TMPDIR/installer-$ARCH/$VERSION/images/MD5SUMS" ]; then
+ (
+ cd "$TMPDIR/installer-$ARCH/$VERSION/images"
+ md5sum -c --status MD5SUMS || error "Error while checking MD5SUMS"
+ )
+else
+ error "Missing MD5SUMS file"
+fi
+
+# Move the data to the final location
+mv "$TMPDIR/installer-$ARCH/$VERSION" "$TARGET"
+mv "$TMPDIR/installer-$ARCH/current" "$TARGET"
+
+# Fixup permissions
+find "$TARGET/$VERSION" -type d -exec chmod 755 {} +
+find "$TARGET/$VERSION" -type f -exec chmod 644 {} +
+
+trap - EXIT
+cleanup
+
+exit 0
--- /dev/null
+#!/usr/bin/python
+
+# Copyright (C) 2007 Florian Reitmeir <florian@reitmeir.org>
+# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
+
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# requires: python-dateutil
+
+import glob, os, sys
+import time, datetime
+import re
+from datetime import datetime
+from datetime import timedelta
+from optparse import OptionParser
+
+RULES = [
+ {'days':14, 'interval':0},
+ {'days':31, 'interval':7},
+ {'days':365, 'interval':31},
+ {'days':3650, 'interval':365},
+
+ # keep 14 days, all each day
+ # keep 31 days, 1 each 7th day
+ # keep 365 days, 1 each 31th day
+]
+
+TODAY = datetime.today()
+VERBOSE = False
+NOACTION = False
+PRINT = False
+PREFIX = ''
+PATH = ''
+
+def all_files(pattern, search_path, pathsep=os.pathsep):
+ """ Given a search path, yield all files matching the pattern. """
+ for path in search_path.split(pathsep):
+ for match in glob.glob(os.path.join(path, pattern)):
+ yield match
+
+def parse_file_dates(list):
+ out = []
+ # dump_2006.05.02-11:52:01.bz2
+ p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+ for file in list:
+ m = p.search(file)
+ if m:
+ d = datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6)))
+ out.append({'name': file, 'date': d})
+ return out
+
+def prepare_rules(rules):
+ out = []
+ for rule in rules:
+ out.append(
+ {
+ 'days':timedelta(days=rule['days']),
+ 'interval':timedelta(days=rule['interval'])}
+ )
+ return out
+
+def expire(rules, list):
+ t_rules=prepare_rules(rules)
+ rule = t_rules.pop(0)
+ last = list.pop(0)
+
+ for file in list:
+ if VERBOSE:
+ print "current file to expire: " + file['name']
+ print file['date']
+
+ # check if rule applies
+ if (file['date'] < (TODAY-rule['days'])):
+ if VERBOSE:
+ print "move to next rule"
+ if t_rules:
+ rule = t_rules.pop(0)
+
+ if (last['date'] - file['date']) < rule['interval']:
+ if VERBOSE:
+ print "unlink file:" + file['name']
+ if PRINT:
+ print file['name']
+ if not NOACTION:
+ os.unlink(file['name'])
+ else:
+ last = file
+ if VERBOSE:
+ print "kept file:" + file['name']
+
+
+parser = OptionParser()
+parser.add_option("-d", "--directory", dest="directory",
+ help="directory name", metavar="Name")
+parser.add_option("-f", "--pattern", dest="pattern",
+ help="Pattern maybe some glob", metavar="*.backup")
+parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+ help="verbose")
+parser.add_option("-n", "--no-action", action="store_true", dest="noaction", default=False,
+ help="just prints what would be done, this implies verbose")
+parser.add_option("-p", "--print", action="store_true", dest="printfiles", default=False,
+ help="just print the filenames that should be deleted, this forbids verbose")
+
+(options, args) = parser.parse_args()
+
+if (not options.directory):
+ parser.error("no directory to check given")
+
+if options.noaction:
+ VERBOSE=True
+ NOACTION=True
+
+if options.verbose:
+ VERBOSE=True
+
+if options.printfiles:
+ VERBOSE=False
+ PRINT=True
+
+files = sorted( list(all_files(options.pattern,options.directory)), reverse=True );
+
+if not files:
+ sys.exit(0)
+
+files_dates = parse_file_dates(files);
+expire(RULES, files_dates)
CREATE INDEX binaries_fingerprint on binaries (sig_fpr);
CREATE INDEX source_fingerprint on source (sig_fpr);
CREATE INDEX dsc_files_file ON dsc_files (file);
+
+-- Own function
+CREATE FUNCTION space_concat(text, text) RETURNS text
+ AS $_$select case
+WHEN $2 is null or $2 = '' THEN $1
+WHEN $1 is null or $1 = '' THEN $2
+ELSE $1 || ' ' || $2
+END$_$
+ LANGUAGE sql;
+
+CREATE AGGREGATE space_separated_list (
+ BASETYPE = text,
+ SFUNC = space_concat,
+ STYPE = text,
+ INITCOND = ''
+);
projects / dak.git / commitdiff