# Installs Debian packaes
# Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
-# $Id: katie,v 1.35 2001-03-24 03:30:16 troup Exp $
+# $Id: katie,v 1.50 2001-06-24 23:17:43 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
#########################################################################################
-import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time
+import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
import apt_inst, apt_pkg
import utils, db_access
###############################################################################
-re_isanum = re.compile (r'^\d+$');
-re_changes = re.compile (r'changes$');
+re_isanum = re.compile (r"^\d+$");
+re_changes = re.compile (r"changes$");
re_default_answer = re.compile(r"\[(.*)\]");
re_fdnic = re.compile("\n\n");
re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
+re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
+re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
#########################################################################################
orig_tar_location = "";
legacy_source_untouchable = {};
Subst = {};
+nmu = None;
#########################################################################################
return 0
return 1
-#####################################################################################################################
+######################################################################################################
+
+class nmu_p:
+ # Read in the group maintainer override file
+ def __init__ (self):
+ self.group_maint = {};
+ if Cnf.get("Dinstall::GroupOverrideFilename"):
+ filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
+ file = utils.open_file(filename, 'r');
+ for line in file.readlines():
+ line = string.strip(utils.re_comments.sub('', line));
+ if line != "":
+ self.group_maint[line] = 1;
+ file.close();
+
+ def is_an_nmu (self, changes, dsc):
+ (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
+ # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
+ if dsc_name == changes["maintainername"] and (changes["changedbyname"] == "" or changes["changedbyname"] == dsc_name):
+ return 0;
+
+ if dsc.has_key("uploaders"):
+ uploaders = string.split(dsc["uploaders"], ",");
+ uploadernames = {};
+ for i in uploaders:
+ (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
+ uploadernames[name] = "";
+ if uploadernames.has_key(changes["changedbyname"]):
+ return 0;
+
+ # Some group maintained packages (e.g. Debian QA) are never NMU's
+ if self.group_maint.has_key(changes["maintainername"]):
+ return 0;
+
+ return 1;
+
+######################################################################################################
+
+# Ensure that source exists somewhere in the archive for the binary
+# upload being processed.
+#
+# (1) exact match => 1.0-3
+# (2) Bin-only NMU of an MU => 1.0-3.0.1
+# (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
+
+def source_exists (package, source_version):
+ q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
+
+ # Reduce the query results to a list of version numbers
+ ql = map(lambda x: x[0], q.getresult());
+
+ # Try (1)
+ if ql.count(source_version):
+ return 1;
+
+ # Try (2)
+ orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
+ if ql.count(orig_source_version):
+ return 1;
+
+ # Try (3)
+ orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
+ if ql.count(orig_source_version):
+ return 1;
+
+ # No source found...
+ return 0;
+
+######################################################################################################
# See if a given package is in the override table
global reject_message, changes, files
# Default in case we bail out
- changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
-
+ changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
+ changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
+ changes["architecture"] = {};
+
# Parse the .changes field into a dictionary
try:
changes = utils.parse_changes(filename, 0)
# Fix the Changed-By: field to be RFC822 compatible; if it exists.
(changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
- # For source uploads the Changed-By field wins; otherwise Maintainer wins.
- if changes["architecture"].has_key("source"):
- changes["uploader822"] = "To: %s\nCc: %s" % (changes["changedby822"], changes["maintainer822"]);
- # changes["uploadername"], changes["uploaderemail"]) = (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]);
-
# Ensure all the values in Closes: are numbers
if changes.has_key("closes"):
for i in changes["closes"].keys():
# FIXME: should probably remove anything that != stable
for i in ("frozen", "unstable"):
if changes["distribution"].has_key(i):
- reject_message = reject_message + "Removing %s from distribution list.\n"
+ reject_message = reject_message + "Removing %s from distribution list.\n" % (i)
del changes["distribution"][i]
changes["stable upload"] = 1;
# If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
for file in files.keys():
# Check the file is readable
if os.access(file,os.R_OK) == 0:
- reject_message = reject_message + "Rejected: Can't read `%s'.\n" % (file)
+ if os.path.exists(file):
+ reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
+ else:
+ reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
+
files[file]["type"] = "unreadable";
continue
# If it's byhand skip remaining checks
files[file]["source"] = control.Find("Source", "");
if files[file]["source"] == "":
files[file]["source"] = files[file]["package"];
+ # Get the source version
+ source = files[file]["source"];
+ source_version = ""
+ if string.find(source, "(") != -1:
+ m = utils.re_extract_src_version.match(source)
+ source = m.group(1)
+ source_version = m.group(2)
+ if not source_version:
+ source_version = files[file]["version"];
+ files[file]["source package"] = source;
+ files[file]["source version"] = source_version;
+
# Checks for a source package...
else:
m = utils.re_issource.match(file)
if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
files[file]["new"] = 1
- # Find any old binary packages
if files[file]["type"] == "deb":
+ # Find any old binary packages
q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND a.arch_string = '%s' AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
% (files[file]["package"], suite, files[file]["architecture"]))
oldfiles = q.dictresult()
if q.getresult() != []:
reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
+ # Check for existent source
+ # FIXME: this is no longer per suite
+ if changes["architecture"].has_key("source"):
+ source_version = files[file]["source version"];
+ if source_version != changes["version"]:
+ reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
+ else:
+ if not source_exists (files[file]["source package"], source_version):
+ reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
+
# Find any old .dsc files
elif files[file]["type"] == "dsc":
q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
# Not there? Check in Incoming...
# [See comment above process_it() for explanation
# of why this is necessary...]
- if os.access(dsc_file, os.R_OK) != 0:
+ if os.path.exists(dsc_file):
files[dsc_file] = {};
files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
def update_subst (changes_filename):
global Subst;
-
+
if changes.has_key("architecture"):
Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
+ else:
+ Subst["__ARCHITECTURE__"] = "Unknown";
Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
- Subst["__FILE_CONTENTS__"] = changes.get("filecontents");
- Subst["__MAINTAINER_ADDRESS__"] = changes["maintainer822"];
- Subst["__MAINTAINER__"] = changes.get("maintainer");
+ Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
+
+ # For source uploads the Changed-By field wins; otherwise Maintainer wins.
+ if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
+ Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
+ Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
+ Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
+ else:
+ Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
+ Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
+ Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
+
Subst["__REJECT_MESSAGE__"] = reject_message;
- Subst["__SOURCE__"] = changes.get("source");
- Subst["__VERSION__"] = changes.get("version");
+ Subst["__SOURCE__"] = changes.get("source", "Unknown");
+ Subst["__VERSION__"] = changes.get("version", "Unknown");
#####################################################################################################################
answer = 'S'
if string.find(reject_message, "Rejected") != -1:
- if time.time()-os.path.getmtime(changes_filename) < 86400:
+ try:
+ modified_time = time.time()-os.path.getmtime(changes_filename);
+ except: # i.e. ignore errors like 'file does not exist';
+ modified_time = 0;
+ if modified_time < 86400:
print "SKIP (too new)\n" + reject_message,;
prompt = "[S]kip, Manual reject, Quit ?";
else:
architecture_id = db_access.get_architecture_id (architecture);
type = files[file]["dbtype"];
dsc_component = files[file]["component"]
- source = files[file]["source"]
- source_version = ""
- if string.find(source, "(") != -1:
- m = utils.re_extract_src_version.match(source)
- source = m.group(1)
- source_version = m.group(2)
- if not source_version:
- source_version = version
+ source = files[file]["source package"]
+ source_version = files[file]["source version"];
filename = files[file]["pool name"] + file;
if not files[file]["files id"]:
files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
try:
utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
except:
- sys.stderr.write("W: couldn't move changes file '%s' to DONE directory [Got %s].\n" % (os.path.basename(changes_filename), sys.exc_type));
+ utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
install_count = install_count + 1;
q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
ql = q.getresult()
if ql == []:
- sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s) in source table.\n" % (package, version));
- sys.exit(1);
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
source_id = ql[0][0];
suite_id = db_access.get_suite_id('proposed-updates');
projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
ql = q.getresult()
if ql == []:
- sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s for %s architecture) in binaries table.\n" % (package, version, architecture));
- sys.exit(1);
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
binary_id = ql[0][0];
suite_id = db_access.get_suite_id('proposed-updates');
projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
projectB.query("COMMIT WORK");
- utils.move (changes_filename, Cnf["Rhona::Morgue"] + os.path.basename(changes_filename));
+ # FIXME
+ utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
# Update the Stable ChangeLog file
if not Cnf["Dinstall::Options::No-Mail"]:
Subst["__SUITE__"] = " into stable";
Subst["__SUMMARY__"] = summary;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
utils.send_mail (mail_message, "")
announce (short_summary, 1)
try:
utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
except:
- sys.stderr.write("W: couldn't reject changes file '%s' [Got %s].\n" % (base_changes_filename, sys.exc_type));
+ utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
pass;
for file in files.keys():
if os.path.exists(file):
try:
utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
except:
- sys.stderr.write("W: couldn't reject file '%s' [Got %s].\n" % (file, sys.exc_type));
+ utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
pass;
# If this is not a manual rejection generate the .reason file and rejection mail message
os.write(fd, reject_message);
os.close(fd);
Subst["__MANUAL_REJECT_MESSAGE__"] = "";
+ Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
else: # Have a manual rejection file to use
reject_mail_message = ""; # avoid <undef>'s
global Subst;
# Build up the rejection email
- user_email_address = string.replace(string.split(pwd.getpwuid(os.getuid())[4],',')[0], '.', '')
- user_email_address = user_email_address + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
+ user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
manual_reject_message = Cnf.get("Dinstall::Options::Manual-Reject", "")
Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
+ Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
# Write the rejection email out as the <foo>.reason file
# If we weren't given one, spawn an editor so the user can add one in
if manual_reject_message == "":
- result = os.system("vi +6 %s" % (reject_file))
+ result = os.system("vi +6 %s" % (reject_filename))
if result != 0:
- sys.stderr.write ("vi invocation failed for `%s'!\n" % (reject_file))
- sys.exit(result)
+ utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
# Then process it as if it were an automatic rejection
reject (changes_filename, reject_filename)
mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
utils.send_mail (mail_message, "")
- (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
bugs = changes["closes"].keys()
bugs.sort()
- # changes["changedbyname"] == dsc_name is probably never true, but better
- # safe than sorry
- if dsc_name == changes["maintainername"] and (changes["changedbyname"] == "" or changes["changedbyname"] == dsc_name):
+ if not nmu.is_an_nmu(changes, dsc):
summary = summary + "Closing bugs: "
for bug in bugs:
summary = summary + "%s " % (bug)
# save and restore it.
cwd = os.getcwd();
- check_signature (changes_file);
- check_changes (changes_file);
- while reprocess:
- reprocess = 0;
- check_files ();
- check_md5sums ();
- check_dsc ();
- check_diff ();
+ try:
+ check_signature (changes_file);
+ check_changes (changes_file);
+ while reprocess:
+ reprocess = 0;
+ check_files ();
+ check_md5sums ();
+ check_dsc ();
+ check_diff ();
+ except:
+ print "ERROR";
+ traceback.print_exc(file=sys.stdout);
+ pass;
update_subst(changes_file);
action(changes_file);
###############################################################################
def main():
- global Cnf, projectB, install_bytes, new_ack_old, Subst
+ global Cnf, projectB, install_bytes, new_ack_old, Subst, nmu
apt_pkg.init();
# Check that we aren't going to clash with the daily cron job
if os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Cnf["Dinstall::Options::No-Lock"]:
- sys.stderr.write("Archive maintenance in progress. Try again later.\n");
- sys.exit(2);
+ utils.fubar("Archive maintenance in progress. Try again later.");
# Obtain lock if not in no-action mode
Subst = {}
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
- bcc = "X-Katie: $Revision: 1.35 $"
+ bcc = "X-Katie: $Revision: 1.50 $"
if Cnf.has_key("Dinstall::Bcc"):
Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
else:
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
+ # Read in the group-maint override file
+ nmu = nmu_p();
+
+ # Sort the .changes files so that we process sourceful ones first
+ changes_files.sort(utils.changes_compare);
+
# Process the changes files
for changes_file in changes_files:
print "\n" + changes_file;
projects / dak.git / blobdiff