#!/usr/bin/env python
# Utility functions for katie
-# Copyright (C) 2001, 2002, 2003 James Troup <james@nocrew.org>
-# $Id: katie.py,v 1.39 2003-09-16 20:41:21 troup Exp $
+# Copyright (C) 2001, 2002, 2003, 2004 James Troup <james@nocrew.org>
+# $Id: katie.py,v 1.50 2004-11-27 16:07:07 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
###############################################################################
-import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
+import cPickle, errno, os, pg, re, stat, string, sys, time;
import utils, db_access;
import apt_inst, apt_pkg;
changes = pkg.changes;
dsc = pkg.dsc;
- (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]).lower());
+ i = utils.fix_maintainer (dsc.get("maintainer",
+ Cnf["Dinstall::MyEmailAddress"]).lower());
+ (dsc_rfc822, dsc_rfc2047, dsc_name, dsc_email) = i;
# changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
if dsc_name == changes["maintainername"].lower() and \
(changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name):
uploaders = dsc["uploaders"].lower().split(",");
uploadernames = {};
for i in uploaders:
- (rfc822, name, email) = utils.fix_maintainer (i.strip());
+ (rfc822, rfc2047, name, email) = utils.fix_maintainer (i.strip());
uploadernames[name] = "";
if uploadernames.has_key(changes["changedbyname"].lower()):
return 0;
def __init__(self, Cnf):
self.Cnf = Cnf;
- self.values = {};
# Read in the group-maint override file
self.nmu = nmu_p(Cnf);
self.accept_count = 0;
exec "self.pkg.%s.clear();" % (i);
self.pkg.orig_tar_id = None;
self.pkg.orig_tar_location = "";
+ self.pkg.orig_tar_gz = None;
###########################################################################
d_files[file][i] = files[file][i];
## changes
# Mandatory changes fields
- for i in [ "distribution", "source", "architecture", "version", "maintainer",
- "urgency", "fingerprint", "changedby822", "changedbyname",
- "maintainername", "maintaineremail", "closes" ]:
+ for i in [ "distribution", "source", "architecture", "version",
+ "maintainer", "urgency", "fingerprint", "changedby822",
+ "changedby2047", "changedbyname", "maintainer822",
+ "maintainer2047", "maintainername", "maintaineremail",
+ "closes", "changes" ]:
d_changes[i] = changes[i];
# Optional changes fields
- # FIXME: changes should be mandatory
- for i in [ "changed-by", "maintainer822", "filecontents", "format",
- "changes", "lisa note" ]:
+ for i in [ "changed-by", "filecontents", "format", "lisa note" ]:
if changes.has_key(i):
d_changes[i] = changes[i];
## dsc
- for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
+ for i in [ "source", "version", "maintainer", "fingerprint",
+ "uploaders", "bts changelog" ]:
if dsc.has_key(i):
d_dsc[i] = dsc[i];
## dsc_files
# If jennifer crashed out in the right place, architecture may still be a string.
if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
changes["architecture"] = { "Unknown" : "" };
- # and maintainer822 may not exist.
- if not changes.has_key("maintainer822"):
- changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"];
+ # and maintainer2047 may not exist.
+ if not changes.has_key("maintainer2047"):
+ changes["maintainer2047"] = self.Cnf["Dinstall::MyEmailAddress"];
Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys());
Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file);
# For source uploads the Changed-By field wins; otherwise Maintainer wins.
if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
- Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
- Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
+ Subst["__MAINTAINER_FROM__"] = changes["changedby2047"];
+ Subst["__MAINTAINER_TO__"] = "%s, %s" % (changes["changedby2047"],
+ changes["maintainer2047"]);
Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
else:
- Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
- Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
+ Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"];
+ Subst["__MAINTAINER_TO__"] = changes["maintainer2047"];
Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"])
if files[file]["type"] == "deb":
summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
else:
- files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
+ files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"])
destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
summary += file + "\n to " + destination + "\n"
Subst["__STABLE_WARNING__"] = "";
mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close");
utils.send_mail (mail_message);
- if action:
- self.Logger.log(["closing bugs"]+bugs);
+ if action:
+ self.Logger.log(["closing bugs"]+bugs);
else: # NMU
summary += "Setting bugs to severity fixed: ";
Cnf = self.Cnf;
Subst = self.Subst;
files = self.pkg.files;
+ changes = self.pkg.changes;
+ changes_file = self.pkg.changes_file;
+ dsc = self.pkg.dsc;
print "Accepting."
- self.Logger.log(["Accepting changes",self.pkg.changes_file]);
+ self.Logger.log(["Accepting changes",changes_file]);
self.dump_vars(Cnf["Dir::Queue::Accepted"]);
# Move all the files into the accepted directory
- utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
+ utils.move(changes_file, Cnf["Dir::Queue::Accepted"]);
file_keys = files.keys();
for file in file_keys:
utils.move(file, Cnf["Dir::Queue::Accepted"]);
utils.send_mail(mail_message)
self.announce(short_summary, 1)
- # Special support to enable clean auto-building of accepted packages
+
+ ## Helper stuff for DebBugs Version Tracking
+ if Cnf.Find("Dir::Queue::BTSVersionTrack"):
+ # ??? once queue/* is cleared on *.d.o and/or reprocessed
+ # the conditionalization on dsc["bts changelog"] should be
+ # dropped.
+
+ # Write out the version history from the changelog
+ if changes["architecture"].has_key("source") and \
+ dsc.has_key("bts changelog"):
+
+ temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
+ dotprefix=1, perms=0644);
+ version_history = utils.open_file(temp_filename, 'w');
+ version_history.write(dsc["bts changelog"]);
+ version_history.close();
+ filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
+ changes_file[:-8]+".versions");
+ os.rename(temp_filename, filename);
+
+ # Write out the binary -> source mapping.
+ temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
+ dotprefix=1, perms=0644);
+ debinfo = utils.open_file(temp_filename, 'w');
+ for file in file_keys:
+ f = files[file];
+ if f["type"] == "deb":
+ line = " ".join([f["package"], f["version"],
+ f["architecture"], f["source package"],
+ f["source version"]]);
+ debinfo.write(line+"\n");
+ debinfo.close();
+ filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
+ changes_file[:-8]+".debinfo");
+ os.rename(temp_filename, filename);
+
+ ## Special support to enable clean auto-building of accepted packages
self.projectB.query("BEGIN WORK");
- for suite in self.pkg.changes["distribution"].keys():
+ for suite in changes["distribution"].keys():
if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
continue;
suite_id = db_access.get_suite_id(suite);
return;
summary = "";
- for file in files.keys():
+ file_keys = files.keys();
+ file_keys.sort();
+ for file in file_keys:
if not files[file].has_key("new") and files[file]["type"] == "deb":
section = files[file]["section"];
override_section = files[file]["override section"];
# If we weren't given a manual rejection message, spawn an
# editor so the user can add one in...
if manual and not reject_message:
- temp_filename = tempfile.mktemp();
- fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
- os.close(fd);
+ temp_filename = utils.temp_filename();
editor = os.environ.get("EDITOR","vi")
answer = 'E';
while answer == 'E':
pkg = self.pkg;
reason_filename = pkg.changes_file[:-8] + ".reason";
- reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
+ reason_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename;
# Move all the files into the reject directory
reject_files = pkg.files.keys() + [pkg.changes_file];
# If we fail here someone is probably trying to exploit the race
# so let's just raise an exception ...
- if os.path.exists(reject_filename):
- os.unlink(reject_filename);
- fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
+ if os.path.exists(reason_filename):
+ os.unlink(reason_filename);
+ reason_file = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
if not manual:
Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
Subst["__MANUAL_REJECT_MESSAGE__"] = "";
Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
- os.write(fd, reject_message);
- os.close(fd);
+ os.write(reason_file, reject_message);
reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
else:
# Build up the rejection email
Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message;
Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected");
-
# Write the rejection email out as the <foo>.reason file
- os.write(fd, reject_mail_message);
- os.close(fd);
+ os.write(reason_file, reject_mail_message);
+
+ os.close(reason_file);
# Send the rejection mail if appropriate
if not Cnf["Dinstall::Options::No-Mail"]:
maps = self.Cnf.ValueList("SuiteMappings")[:]
maps.reverse()
maps = [ m.split() for m in maps ]
- maps = [ (x[1], x[2]) for x in maps
+ maps = [ (x[1], x[2]) for x in maps
if x[0] == "map" or x[0] == "silent-map" ]
s = [suite]
for x in maps:
if x[1] in s and x[0] not in s:
s.append(x[0])
-
+
que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR "));
q = self.projectB.query(que)
ql = map(lambda x: x[0], q.getresult());
# Try (1)
- if ql.count(source_version):
+ if source_version in ql:
continue
# Try (2)
orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version)
- if ql.count(orig_source_version):
+ if orig_source_version in ql:
continue
# Try (3)
orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version)
- if ql.count(orig_source_version):
+ if orig_source_version in ql:
continue
# No source found...
return okay
################################################################################
-
+
def in_override_p (self, package, component, suite, binary_type, file):
files = self.pkg.files;
type_id = db_access.get_override_type_id(type);
# FIXME: nasty non-US speficic hack
- if component[:7].lower() == "non-us/":
+ if component.lower().startswith("non-us/"):
component = component[7:];
q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
existent_version = entry[0];
suite = entry[1];
if suite in must_be_newer_than and \
- apt_pkg.VersionCompare(new_version, existent_version) != 1:
+ apt_pkg.VersionCompare(new_version, existent_version) < 1:
self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
if suite in must_be_older_than and \
- apt_pkg.VersionCompare(new_version, existent_version) != -1:
+ apt_pkg.VersionCompare(new_version, existent_version) > -1:
self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite));
################################################################################
files = self.pkg.files;
dsc_files = self.pkg.dsc_files;
legacy_source_untouchable = self.pkg.legacy_source_untouchable;
- orig_tar_gz = None;
+ self.pkg.orig_tar_gz = None;
# Try and find all files mentioned in the .dsc. This has
# to work harder to cope with the multiple possible
actual_size = int(files[dsc_file]["size"]);
found = "%s in incoming" % (dsc_file)
# Check the file does not already exist in the archive
- q = self.projectB.query("SELECT size, md5sum, filename FROM files WHERE filename LIKE '%%%s%%'" % (dsc_file));
-
+ q = self.projectB.query("SELECT f.size, f.md5sum, l.path, f.filename FROM files f, location l WHERE f.filename LIKE '%%%s%%' AND l.id = f.location" % (dsc_file));
ql = q.getresult();
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
- if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file:
- self.Logger.log(["check_dsc_against_db",i[2],dsc_file]);
+ if i[3] != dsc_file and i[3][-(len(dsc_file)+1):] != '/'+dsc_file:
ql.remove(i);
# "[katie] has not broken them. [katie] has fixed a
files[dsc_file]["md5sum"] == i[1]:
self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: ");
del files[dsc_file];
+ self.pkg.orig_tar_gz = i[2] + i[3];
match = 1;
if not match:
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
- self.Logger.log(["check_dsc_against_db",i[1],dsc_file]);
ql.remove(i);
if ql:
- # Unfortunately, we make get more than one
- # match here if, for example, the package was
- # in potato but had a -sa upload in woody. So
- # we need to choose the right one.
+ # Unfortunately, we may get more than one match here if,
+ # for example, the package was in potato but had an -sa
+ # upload in woody. So we need to choose the right one.
x = ql[0]; # default to something sane in case we don't match any or have only one
dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
# See install() in katie...
self.pkg.orig_tar_id = x[3];
+ self.pkg.orig_tar_gz = old_file;
if suite_type == "legacy" or suite_type == "legacy-mixed":
self.pkg.orig_tar_location = "legacy";
else:
actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
found = in_otherdir;
+ self.pkg.orig_tar_gz = in_otherdir;
if not found:
self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
+ self.pkg.orig_tar_gz = -1;
continue;
else:
self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
if actual_size != int(dsc_files[dsc_file]["size"]):
self.reject("size for %s doesn't match %s." % (found, file));
- return (self.reject_message, orig_tar_gz);
+ return (self.reject_message, None);
def do_query(self, q):
sys.stderr.write("query: \"%s\" ... " % (q));