#!/usr/bin/env python
-# Installs Debian packaes
-# Copyright (C) 2000 James Troup <james@nocrew.org>
-# $Id: katie,v 1.18 2001-01-16 21:52:37 troup Exp $
+# Installs Debian packages
+# Copyright (C) 2000, 2001 James Troup <james@nocrew.org>
+# $Id: katie,v 1.68 2002-01-28 18:53:01 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-# Based (almost entirely) on dinstall by Guy Maor <maor@debian.org>
+# Originally based almost entirely on dinstall by Guy Maor <maor@debian.org>
-#########################################################################################
+###############################################################################
# Cartman: "I'm trying to make the best of a bad situation, I don't
# need to hear crap from a bunch of hippy freaks living in
#
# Cartman: "uhh.. screw you guys... home."
-#########################################################################################
+###############################################################################
-import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time
+import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback
import apt_inst, apt_pkg
-import utils, db_access
+import utils, db_access, logging
+
+from types import *;
###############################################################################
-re_isanum = re.compile (r'^\d+$');
-re_isadeb = re.compile (r'.*\.u?deb$');
-re_issource = re.compile (r'(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)');
-re_changes = re.compile (r'changes$');
+re_isanum = re.compile (r"^\d+$");
+re_changes = re.compile (r"changes$");
re_default_answer = re.compile(r"\[(.*)\]");
re_fdnic = re.compile("\n\n");
re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null");
-
-###############################################################################
-
-#
-reject_footer = """If you don't understand why your files were rejected, or if the
-override file requires editing, reply to this email.
-
-Your rejected files are in incoming/REJECT/. (Some may also be in
-incoming/ if your .changes file was unparsable.) If only some of the
-files need to repaired, you may move any good files back to incoming/.
-Please remove any bad files from incoming/REJECT/."""
-#
-new_ack_footer = """Your package contains new components which requires manual editing of
-the override file. It is ok otherwise, so please be patient. New
-packages are usually added to the override file about once a week.
-
-You may have gotten the distribution wrong. You'll get warnings above
-if files already exist in other distributions."""
-#
-installed_footer = """If the override file requires editing, file a bug on ftp.debian.org.
-
-Thank you for your contribution to Debian GNU."""
+re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$");
+re_bin_only_nmu_of_nmu = re.compile("\.\d+$");
#########################################################################################
# Globals
Cnf = None;
+Options = None;
+Logger = None;
reject_message = "";
changes = {};
dsc = {};
install_bytes = 0.0;
reprocess = 0;
orig_tar_id = None;
+orig_tar_location = "";
legacy_source_untouchable = {};
+Subst = {};
+nmu = None;
+katie_version = "$Revision: 1.68 $";
-#########################################################################################
+###############################################################################
+
+def init():
+ global Cnf, Options;
-def usage (exit_code):
+ Cnf = utils.get_conf()
+
+ Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
+ ('h',"help","Dinstall::Options::Help"),
+ ('k',"ack-new","Dinstall::Options::Ack-New"),
+ ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
+ ('n',"no-action","Dinstall::Options::No-Action"),
+ ('p',"no-lock", "Dinstall::Options::No-Lock"),
+ ('s',"no-mail", "Dinstall::Options::No-Mail"),
+ ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
+ ('V',"version","Dinstall::Options::Version")];
+
+ for i in ["automatic", "help", "ack-new", "manual-reject", "no-action",
+ "no-lock", "no-mail", "override-distribution", "version"]:
+ if not Cnf.has_key("Dinstall::Options::%s" % (i)):
+ Cnf["Dinstall::Options::%s" % (i)] = "";
+
+ changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
+ Options = Cnf.SubTree("Dinstall::Options")
+
+ return changes_files;
+
+###############################################################################
+
+def usage (exit_code=0):
print """Usage: dinstall [OPTION]... [CHANGES]...
-a, --automatic automatic run
- -d, --debug=VALUE debug
- -k, --ack-new acknowledge new packages
+ -h, --help show this help and exit.
+ -k, --ack-new acknowledge new packages !! for cron.daily only !!
-m, --manual-reject=MSG manual reject with `msg'
- -n, --dry-run don't do anything
+ -n, --no-action don't do anything
-p, --no-lock don't check lockfile !! for cron.daily only !!
- -r, --no-version-check override version check
- -u, --distribution=DIST override distribution to `dist'"""
+ -s, --no-mail don't send any mail
+ -u, --distribution=DIST override distribution to `dist'
+ -V, --version display the version number and exit"""
sys.exit(exit_code)
+#########################################################################################
+
def check_signature (filename):
global reject_message
(result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename))
if (result != 0):
- reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (filename, output)
+ reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output)
return 0
return 1
-#####################################################################################################################
+######################################################################################################
+
+class nmu_p:
+ # Read in the group maintainer override file
+ def __init__ (self):
+ self.group_maint = {};
+ if Cnf.get("Dinstall::GroupOverrideFilename"):
+ filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"];
+ file = utils.open_file(filename);
+ for line in file.readlines():
+ line = string.strip(utils.re_comments.sub('', line));
+ if line != "":
+ self.group_maint[line] = 1;
+ file.close();
+
+ def is_an_nmu (self, changes, dsc):
+ (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
+ # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry
+ if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name):
+ return 0;
+
+ if dsc.has_key("uploaders"):
+ uploaders = string.split(dsc["uploaders"], ",");
+ uploadernames = {};
+ for i in uploaders:
+ (rfc822, name, email) = utils.fix_maintainer (string.strip(i));
+ uploadernames[name] = "";
+ if uploadernames.has_key(changes["changedbyname"]):
+ return 0;
+
+ # Some group maintained packages (e.g. Debian QA) are never NMU's
+ if self.group_maint.has_key(changes["maintaineremail"]):
+ return 0;
+
+ return 1;
+
+######################################################################################################
+
+# Ensure that source exists somewhere in the archive for the binary
+# upload being processed.
+#
+# (1) exact match => 1.0-3
+# (2) Bin-only NMU of an MU => 1.0-3.0.1
+# (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1
+
+def source_exists (package, source_version):
+ q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package));
+
+ # Reduce the query results to a list of version numbers
+ ql = map(lambda x: x[0], q.getresult());
+
+ # Try (1)
+ if ql.count(source_version):
+ return 1;
+
+ # Try (2)
+ orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version);
+ if ql.count(orig_source_version):
+ return 1;
+
+ # Try (3)
+ orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version);
+ if ql.count(orig_source_version):
+ return 1;
+
+ # No source found...
+ return 0;
+
+######################################################################################################
# See if a given package is in the override table
-def in_override_p (package, component, suite, binary_type):
- global overrides;
+def in_override_p (package, component, suite, binary_type, file):
+ global files;
if binary_type == "": # must be source
type = "dsc";
if string.lower(component[:7]) == "non-us/":
component = component[7:];
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s"
+ q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
% (package, suite_id, component_id, type_id));
result = q.getresult();
# If checking for a source package fall back on the binary override type
if type == "dsc" and not result:
type_id = db_access.get_override_type_id("deb");
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s"
+ q = projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id"
% (package, suite_id, component_id, type_id));
result = q.getresult();
-
+
+ # Remember the section and priority so we can check them later if appropriate
+ if result != []:
+ files[file]["override section"] = result[0][0];
+ files[file]["override priority"] = result[0][1];
+
return result;
#####################################################################################################################
def check_changes(filename):
global reject_message, changes, files
- # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.]
+ # Default in case we bail out
+ changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
+ changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"];
+ changes["architecture"] = {};
+
+ # Parse the .changes field into a dictionary
try:
- changes = utils.parse_changes(filename)
+ changes = utils.parse_changes(filename, 0)
except utils.cant_open_exc:
- reject_message = "Rejected: can't read changes file '%s'.\n" % (filename)
+ reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
return 0;
except utils.changes_parse_error_exc, line:
- reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
- changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
+ reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
return 0;
- # Parse the Files field from the .changes into another dictionary [FIXME need to trap errors as above]
+ # Parse the Files field from the .changes into another dictionary
try:
files = utils.build_file_list(changes, "");
except utils.changes_parse_error_exc, line:
- reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
+ reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line);
+ except utils.nk_format_exc, format:
+ reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename);
+ return 0;
# Check for mandatory fields
- for i in ("source", "binary", "architecture", "version", "distribution","maintainer", "files"):
+ for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"):
if not changes.has_key(i):
- reject_message = "Rejected: Missing field `%s' in changes file." % (i)
+ reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i)
return 0 # Avoid <undef> errors during later tests
- # Fix the Maintainer: field to be RFC822 compatible
- (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
-
# Override the Distribution: field if appropriate
- if Cnf["Dinstall::Options::Override-Distribution"] != "":
- reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"])
- changes["distribution"] = Cnf["Dinstall::Options::Override-Distribution"]
+ if Options["Override-Distribution"] != "":
+ reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"])
+ changes["distribution"] = Options["Override-Distribution"]
# Split multi-value fields into a lower-level dictionary
for i in ("architecture", "distribution", "binary", "closes"):
for j in string.split(o):
changes[i][j] = 1
+ # Fix the Maintainer: field to be RFC822 compatible
+ (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"])
+
+ # Fix the Changed-By: field to be RFC822 compatible; if it exists.
+ (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by",""));
+
# Ensure all the values in Closes: are numbers
if changes.has_key("closes"):
for i in changes["closes"].keys():
if re_isanum.match (i) == None:
reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i)
+ # Ensure there _is_ a target distribution
+ if not changes["distribution"].keys():
+ reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n";
+
# Map frozen to unstable if frozen doesn't exist
if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"):
del changes["distribution"]["frozen"]
+ changes["distribution"]["unstable"] = 1;
reject_message = reject_message + "Mapping frozen to unstable.\n"
# Map testing to unstable
if changes["distribution"].has_key("testing"):
- del changes["distribution"]["testing"]
- reject_message = reject_message + "Mapping testing to unstable.\n"
+ if len(changes["distribution"].keys()) > 1:
+ del changes["distribution"]["testing"];
+ reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n";
+ else:
+ reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n";
# Ensure target distributions exist
for i in changes["distribution"].keys():
if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)):
reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i)
del changes["distribution"]["stable"]
-
+ changes["distribution"]["unstable"] = 1;
+
# Map arches not being released from frozen to unstable
if changes["distribution"].has_key("frozen"):
for i in changes["architecture"].keys():
if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)):
reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i)
del changes["distribution"]["frozen"]
+ changes["distribution"]["unstable"] = 1;
# Handle uploads to stable
if changes["distribution"].has_key("stable"):
# If running from within proposed-updates; assume an install to stable
if string.find(os.getcwd(), 'proposed-updates') != -1:
- # FIXME: should probably remove anything that != stable
- for i in ("frozen", "unstable"):
- if changes["distribution"].has_key(i):
- reject_message = reject_message + "Removing %s from distribution list.\n"
- del changes["distribution"][i]
- changes["stable upload"] = 1;
+ # Remove non-stable target distributions
+ for dist in changes["distribution"].keys():
+ if dist != "stable":
+ reject_message = reject_message + "Removing %s from distribution list.\n" % (dist);
+ del changes["distribution"][dist];
+ changes["stable install"] = 1;
# If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool
file = files.keys()[0];
if os.access(file, os.R_OK) == 0:
pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]);
+ changes["installing from the pool"] = 1;
os.chdir(pool_dir);
# Otherwise (normal case) map stable to updates
else:
# chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison)
changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"])
changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"])
-
+
if string.find(reject_message, "Rejected:") != -1:
return 0
- else:
+ else:
return 1
def check_files():
global reject_message
-
+
archive = utils.where_am_i();
for file in files.keys():
# Check the file is readable
if os.access(file,os.R_OK) == 0:
- reject_message = reject_message + "Rejected: Can't read `%s'.\n" % (file)
+ if os.path.exists(file):
+ reject_message = reject_message + "Rejected: Can't read `%s'. [permission denied]\n" % (file)
+ else:
+ reject_message = reject_message + "Rejected: Can't read `%s'. [file not found]\n" % (file)
+
files[file]["type"] = "unreadable";
continue
# If it's byhand skip remaining checks
files[file]["byhand"] = 1;
files[file]["type"] = "byhand";
# Checks for a binary package...
- elif re_isadeb.match(file) != None:
+ elif utils.re_isadeb.match(file) != None:
+ files[file]["type"] = "deb";
+
# Extract package information using dpkg-deb
- control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r")))
+ try:
+ control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)));
+ except:
+ reject_message = reject_message + "Rejected: %s: debExtractControl() raised %s.\n" % (file, sys.exc_type);
+ # Can't continue, none of the checks on control would work.
+ continue;
# Check for mandatory fields
if control.Find("Package") == None:
reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file)
if control.Find("Version") == None:
reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file)
-
+
# Ensure the package name matches the one give in the .changes
if not changes["binary"].has_key(control.Find("Package", "")):
reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", ""))
files[file]["dbtype"] = "deb";
else:
reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file);
- files[file]["type"] = "deb";
files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", ""))
files[file]["source"] = control.Find("Source", "");
if files[file]["source"] == "":
files[file]["source"] = files[file]["package"];
+ # Get the source version
+ source = files[file]["source"];
+ source_version = ""
+ if string.find(source, "(") != -1:
+ m = utils.re_extract_src_version.match(source)
+ source = m.group(1)
+ source_version = m.group(2)
+ if not source_version:
+ source_version = files[file]["version"];
+ files[file]["source package"] = source;
+ files[file]["source version"] = source_version;
+
# Checks for a source package...
else:
- m = re_issource.match(file)
+ m = utils.re_issource.match(file)
if m != None:
files[file]["package"] = m.group(1)
files[file]["version"] = m.group(2)
files[file]["type"] = m.group(3)
-
+
# Ensure the source package name matches the Source filed in the .changes
if changes["source"] != files[file]["package"]:
reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"])
check_signature(file)
files[file]["fullname"] = file
+ files[file]["architecture"] = "source";
# Not a binary or source package? Assume byhand...
else:
files[file]["byhand"] = 1;
files[file]["type"] = "byhand";
-
+
files[file]["oldfiles"] = {}
for suite in changes["distribution"].keys():
# Skip byhand
continue
# See if the package is NEW
- if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype","")):
+ if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file):
files[file]["new"] = 1
-
- # Find any old binary packages
+
if files[file]["type"] == "deb":
- q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND a.arch_string = '%s' AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
+ # Find any old binary packages
+ q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id"
% (files[file]["package"], suite, files[file]["architecture"]))
oldfiles = q.dictresult()
for oldfile in oldfiles:
files[file]["oldfiles"][suite] = oldfile
# Check versions [NB: per-suite only; no cross-suite checking done (yet)]
if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1:
- if Cnf["Dinstall::Options::No-Version-Check"]:
- reject_message = reject_message + "Overriden rejection"
- else:
- reject_message = reject_message + "Rejected"
- reject_message = reject_message + ": %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
+ reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"])
# Check for existing copies of the file
- if not changes.has_key("stable upload"):
+ if not changes.has_key("stable install"):
q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"]))
if q.getresult() != []:
reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file)
+ # Check for existent source
+ # FIXME: this is no longer per suite
+ if changes["architecture"].has_key("source"):
+ source_version = files[file]["source version"];
+ if source_version != changes["version"]:
+ reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]);
+ else:
+ if not source_exists (files[file]["source package"], source_version):
+ reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file);
+
# Find any old .dsc files
elif files[file]["type"] == "dsc":
q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file"
if component_id == -1:
reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component);
continue;
-
+
+ # Validate the priority
+ if string.find(files[file]["priority"],'/') != -1:
+ reject_message = reject_message + "Rejected: file '%s' has invalid priority '%s' [contains '/'].\n" % (file, files[file]["priority"]);
+
# Check the md5sum & size against existing files (if any)
location = Cnf["Dir::PoolDir"];
files[file]["location id"] = db_access.get_location_id (location, component, archive);
-
+
files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]);
files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]);
if files_id == -1:
if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]:
files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"];
-
+
if string.find(reject_message, "Rejected:") != -1:
return 0
- else:
+ else:
return 1
###############################################################################
def check_dsc ():
- global dsc, dsc_files, reject_message, reprocess, orig_tar_id, legacy_source_untouchable;
+ global dsc, dsc_files, reject_message, reprocess, orig_tar_id, orig_tar_location, legacy_source_untouchable;
for file in files.keys():
if files[file]["type"] == "dsc":
try:
- dsc = utils.parse_changes(file)
+ dsc = utils.parse_changes(file, 1)
except utils.cant_open_exc:
- reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename)
+ reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (file)
return 0;
except utils.changes_parse_error_exc, line:
- reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line)
+ reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (file, line)
+ return 0;
+ except utils.invalid_dsc_format_exc, line:
+ reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (file, line)
return 0;
try:
dsc_files = utils.build_file_list(dsc, 1)
reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n";
continue;
except utils.changes_parse_error_exc, line:
- reject_message = "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (filename, line);
+ reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line);
continue;
+ # Enforce mandatory fields
+ for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"):
+ if not dsc.has_key(i):
+ reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i)
+
+ # The dpkg maintainer from hell strikes again! Bumping the
+ # version number of the .dsc breaks extraction by stable's
+ # dpkg-source.
+ if dsc["format"] != "1.0":
+ reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version
+ of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version
+ installed.
+""";
+
+ # Ensure the version number in the .dsc matches the version number in the .changes
+ epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version"));
+ changes_version = files[file]["version"];
+ if epochless_dsc_version != files[file]["version"]:
+ reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version);
+
+ # Ensure source is newer than existing source in target suites
+ package = dsc.get("source");
+ new_version = dsc.get("version");
+ for suite in changes["distribution"].keys():
+ q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id"
+ % (package, suite));
+ ql = map(lambda x: x[0], q.getresult());
+ for old_version in ql:
+ if apt_pkg.VersionCompare(new_version, old_version) != 1:
+ reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version)
+
# Try and find all files mentioned in the .dsc. This has
# to work harder to cope with the multiple possible
# locations of an .orig.tar.gz.
actual_size = int(files[dsc_file]["size"]);
found = "%s in incoming" % (dsc_file)
# Check the file does not already exist in the archive
- if not changes.has_key("stable upload"):
+ if not changes.has_key("stable install"):
q = projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
# "It has not broken them. It has fixed a
reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file)
elif dsc_file[-12:] == ".orig.tar.gz":
# Check in the pool
- q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
+ q = projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file));
ql = q.getresult();
-
if ql != []:
# Unfortunately, we make get more than one match
# here if, for example, the package was in potato
if len(ql) > 1:
for i in ql:
old_file = i[0] + i[1];
- actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r"));
+ actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
actual_size = os.stat(old_file)[stat.ST_SIZE];
if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
x = i;
legacy_source_untouchable[i[3]] = "";
old_file = x[0] + x[1];
- actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r"));
+ actual_md5 = apt_pkg.md5sum(utils.open_file(old_file));
actual_size = os.stat(old_file)[stat.ST_SIZE];
found = old_file;
suite_type = x[2];
dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
# See install()...
+ orig_tar_id = x[3];
if suite_type == "legacy" or suite_type == "legacy-mixed":
- orig_tar_id = x[3];
+ orig_tar_location = "legacy";
+ else:
+ orig_tar_location = x[4];
else:
# Not there? Check in Incoming...
# [See comment above process_it() for explanation
# of why this is necessary...]
- if os.access(dsc_file, os.R_OK) != 0:
+ if os.path.exists(dsc_file):
files[dsc_file] = {};
files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE];
files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"];
reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file);
continue;
else:
- reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file);
+ reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming.\n" % (file, dsc_file);
continue;
if actual_md5 != dsc_files[dsc_file]["md5sum"]:
reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file);
if string.find(reject_message, "Rejected:") != -1:
return 0
- else:
+ else:
return 1
###############################################################################
# problem just changed the symptoms.
def check_diff ():
- global dsc, dsc_files, reject_message, reprocess, orig_tar_id;
+ global dsc, dsc_files, reject_message, reprocess;
for filename in files.keys():
if files[filename]["type"] == "diff.gz":
if string.find(reject_message, "Rejected:") != -1:
return 0
- else:
+ else:
return 1
###############################################################################
for file in files.keys():
try:
- file_handle = utils.open_file(file,"r");
+ file_handle = utils.open_file(file);
except utils.cant_open_exc:
pass;
else:
if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]:
reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file);
+def check_override ():
+ global Subst;
+
+ # Only check section & priority on sourceful non-stable installs
+ if not changes["architecture"].has_key("source") or changes.has_key("stable install"):
+ return;
+
+ summary = ""
+ for file in files.keys():
+ if not files[file].has_key("new") and files[file]["type"] == "deb":
+ section = files[file]["section"];
+ override_section = files[file]["override section"];
+ if section != override_section and section != "-":
+ # Ignore this; it's a common mistake and not worth whining about
+ if string.lower(section) == "non-us/main" and string.lower(override_section) == "non-us":
+ continue;
+ summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section);
+ priority = files[file]["priority"];
+ override_priority = files[file]["override priority"];
+ if priority != override_priority and priority != "-":
+ summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority);
+
+ if summary == "":
+ return;
+
+ Subst["__SUMMARY__"] = summary;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.override-disparity","r").read());
+ utils.send_mail (mail_message, "")
+
+#####################################################################################################################
+
+# Set up the per-package template substitution mappings
+
+def update_subst (changes_filename):
+ global Subst;
+
+ # If katie crashed out in the right place, architecture may still be a string.
+ if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType):
+ changes["architecture"] = { "Unknown" : "" };
+ # and maintainer822 may not exist.
+ if not changes.has_key("maintainer822"):
+ changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"];
+
+ Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' );
+ Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename);
+ Subst["__FILE_CONTENTS__"] = changes.get("filecontents", "");
+
+ # For source uploads the Changed-By field wins; otherwise Maintainer wins.
+ if changes["architecture"].has_key("source") and changes["changedby822"] != "" and (changes["changedby822"] != changes["maintainer822"]):
+ Subst["__MAINTAINER_FROM__"] = changes["changedby822"];
+ Subst["__MAINTAINER_TO__"] = changes["changedby822"] + ", " + changes["maintainer822"];
+ Subst["__MAINTAINER__"] = changes.get("changed-by", "Unknown");
+ else:
+ Subst["__MAINTAINER_FROM__"] = changes["maintainer822"];
+ Subst["__MAINTAINER_TO__"] = changes["maintainer822"];
+ Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown");
+ if Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"):
+ Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes.get("source"), Cnf["Dinstall::TrackingServer"])
+
+ Subst["__REJECT_MESSAGE__"] = reject_message;
+ Subst["__SOURCE__"] = changes.get("source", "Unknown");
+ Subst["__VERSION__"] = changes.get("version", "Unknown");
+
#####################################################################################################################
def action (changes_filename):
- byhand = confirm = suites = summary = new = "";
+ byhand = summary = new = "";
# changes["distribution"] may not exist in corner cases
# (e.g. unreadable changes files)
- if not changes.has_key("distribution"):
+ if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
changes["distribution"] = {};
-
+
+ confirm = []
+ suites = []
for suite in changes["distribution"].keys():
if Cnf.has_key("Suite::%s::Confirm"):
- confirm = confirm + suite + ", "
- suites = suites + suite + ", "
- confirm = confirm[:-2]
- suites = suites[:-2]
+ confirm.append(suite)
+ suites.append(suite)
- for file in files.keys():
+ file_keys = files.keys();
+ file_keys.sort();
+ for file in file_keys:
if files[file].has_key("byhand"):
byhand = 1
summary = summary + file + " byhand\n"
if files[file].has_key("othercomponents"):
summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"])
if files[file]["type"] == "deb":
- summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r")))["Description"] + '\n';
+ summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n';
else:
files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"])
destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file
summary = summary + "Changes: " + f;
summary = summary + announce (short_summary, 0)
-
+
(prompt, answer) = ("", "XXX")
- if Cnf["Dinstall::Options::No-Action"] or Cnf["Dinstall::Options::Automatic"]:
+ if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
if string.find(reject_message, "Rejected") != -1:
- if time.time()-os.path.getmtime(changes_filename) < 86400:
+ try:
+ modified_time = time.time()-os.path.getmtime(changes_filename);
+ except: # i.e. ignore errors like 'file does not exist';
+ modified_time = 0;
+ if modified_time < 86400:
print "SKIP (too new)\n" + reject_message,;
prompt = "[S]kip, Manual reject, Quit ?";
else:
print "REJECT\n" + reject_message,;
prompt = "[R]eject, Manual reject, Skip, Quit ?";
- if Cnf["Dinstall::Options::Automatic"]:
+ if Options["Automatic"]:
answer = 'R';
elif new:
- print "NEW to %s\n%s%s" % (suites, reject_message, summary),;
+ print "NEW to %s\n%s%s" % (string.join(suites, ", "), reject_message, summary),;
prompt = "[S]kip, New ack, Manual reject, Quit ?";
- if Cnf["Dinstall::Options::Automatic"] and Cnf["Dinstall::Options::Ack-New"]:
+ if Options["Automatic"] and Options["Ack-New"]:
answer = 'N';
elif byhand:
print "BYHAND\n" + reject_message + summary,;
prompt = "[I]nstall, Manual reject, Skip, Quit ?";
elif confirm:
- print "CONFIRM to %s\n%s%s" % (confirm, reject_message, summary),
+ print "CONFIRM to %s\n%s%s" % (string.join(confirm, ", "), reject_message, summary),
prompt = "[I]nstall, Manual reject, Skip, Quit ?";
else:
print "INSTALL\n" + reject_message + summary,;
prompt = "[I]nstall, Manual reject, Skip, Quit ?";
- if Cnf["Dinstall::Options::Automatic"]:
+ if Options["Automatic"]:
answer = 'I';
while string.find(prompt, answer) == -1:
elif answer == 'Q':
sys.exit(0)
-#####################################################################################################################
+###############################################################################
def install (changes_filename, summary, short_summary):
- global install_count, install_bytes
+ global install_count, install_bytes, Subst;
- # Stable uploads are a special case
- if changes.has_key("stable upload"):
+ # stable installs are a special case
+ if changes.has_key("stable install"):
stable_install (changes_filename, summary, short_summary);
return;
-
+
print "Installing."
+ Logger.log(["installing changes",changes_filename]);
+
archive = utils.where_am_i();
# Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed.
files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id)
projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)"
% (package, version, maintainer_id, files[file]["files id"]))
-
+
for suite in changes["distribution"].keys():
suite_id = db_access.get_suite_id(suite);
projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id))
if files_id == None:
files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id);
projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id));
-
+
# Add the .deb files to the DB
for file in files.keys():
if files[file]["type"] == "deb":
architecture = files[file]["architecture"]
architecture_id = db_access.get_architecture_id (architecture);
type = files[file]["dbtype"];
- component = files[file]["component"]
- source = files[file]["source"]
- source_version = ""
- if string.find(source, "(") != -1:
- m = utils.re_extract_src_version.match(source)
- source = m.group(1)
- source_version = m.group(2)
- if not source_version:
- source_version = version
+ dsc_component = files[file]["component"]
+ source = files[file]["source package"]
+ source_version = files[file]["source version"];
filename = files[file]["pool name"] + file;
if not files[file]["files id"]:
files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"])
# If the .orig.tar.gz is in a legacy directory we need to poolify
# it, so that apt-get source (and anything else that goes by the
# "Directory:" field in the Sources.gz file) works.
- if orig_tar_id != None:
+ if orig_tar_id != None and orig_tar_location == "legacy":
q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id));
qd = q.dictresult();
for qid in qd:
# Then Update the DB's files table
q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"]));
+ # If this is a sourceful diff only upload that is moving non-legacy
+ # cross-component we need to copy the .orig.tar.gz into the new
+ # component too for the same reasons as above.
+ #
+ if changes["architecture"].has_key("source") and orig_tar_id != None and \
+ orig_tar_location != "legacy" and orig_tar_location != dsc_location_id:
+ q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id));
+ ql = q.getresult()[0];
+ old_filename = ql[0] + ql[1];
+ file_size = ql[2];
+ file_md5sum = ql[3];
+ new_filename = utils.poolify (changes["source"], dsc_component) + os.path.basename(old_filename);
+ new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
+ if new_files_id == None:
+ utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename);
+ new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id);
+ projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id));
+
# Install the files into the pool
for file in files.keys():
if files[file].has_key("byhand"):
destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file
destdir = os.path.dirname(destination)
utils.move (file, destination)
+ Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]);
install_bytes = install_bytes + float(files[file]["size"])
# Copy the .changes file across for suite which need it.
projectB.query("COMMIT WORK");
- utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
+ try:
+ utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename))
+ except:
+ utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(changes_filename), sys.exc_type));
install_count = install_count + 1;
- if not Cnf["Dinstall::Options::No-Mail"]:
- mail_message = """Return-Path: %s
-From: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: %s INSTALLED
-
-%s
-Installing:
-%s
-
-%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer)
+ if not Options["No-Mail"]:
+ Subst["__SUITE__"] = "";
+ Subst["__SUMMARY__"] = summary;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
utils.send_mail (mail_message, "")
announce (short_summary, 1)
+ check_override ();
-#####################################################################################################################
+################################################################################
def stable_install (changes_filename, summary, short_summary):
- global install_count, install_bytes
-
+ global install_count, install_bytes, Subst;
+
print "Installing to stable."
archive = utils.where_am_i();
version = dsc["version"] # NB: not files[file]["version"], that has no epoch
q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version))
ql = q.getresult()
- if ql == []:
- sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s) in source table.\n" % (package, version));
- sys.exit(1);
+ if not ql:
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
source_id = ql[0][0];
suite_id = db_access.get_suite_id('proposed-updates');
projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
suite_id = db_access.get_suite_id('stable');
projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id));
-
+ install_bytes = install_bytes + float(files[file]["size"])
+
# Add the .deb files to the DB
for file in files.keys():
if files[file]["type"] == "deb":
architecture = files[file]["architecture"]
q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture))
ql = q.getresult()
- if ql == []:
- sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s for %s architecture) in binaries table.\n" % (package, version, architecture));
- sys.exit(1);
+ if not ql:
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
binary_id = ql[0][0];
suite_id = db_access.get_suite_id('proposed-updates');
projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
suite_id = db_access.get_suite_id('stable');
projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id));
+ install_bytes = install_bytes + float(files[file]["size"])
projectB.query("COMMIT WORK");
- utils.move (changes_filename, Cnf["Rhona::Morgue"] + os.path.basename(changes_filename));
+ # FIXME
+ utils.move (changes_filename, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(changes_filename));
# Update the Stable ChangeLog file
changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog";
if os.path.exists(new_changelog_filename):
os.unlink (new_changelog_filename);
-
+
new_changelog = utils.open_file(new_changelog_filename, 'w');
for file in files.keys():
if files[file]["type"] == "deb":
new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file));
- elif re_issource.match(file) != None:
+ elif utils.re_issource.match(file) != None:
new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file));
else:
new_changelog.write("%s\n" % (file));
chop_changes = re_fdnic.sub("\n", changes["changes"]);
new_changelog.write(chop_changes + '\n\n');
if os.access(changelog_filename, os.R_OK) != 0:
- changelog = utils.open_file(changelog_filename, 'r');
+ changelog = utils.open_file(changelog_filename);
new_changelog.write(changelog.read());
new_changelog.close();
if os.access(changelog_filename, os.R_OK) != 0:
install_count = install_count + 1;
- if not Cnf["Dinstall::Options::No-Mail"]:
- mail_message = """Return-Path: %s
-From: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: %s INSTALLED into stable
-
-%s
-Installing:
-%s
-
-%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer)
+ if not Options["No-Mail"]:
+ Subst["__SUITE__"] = " into stable";
+ Subst["__SUMMARY__"] = summary;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read());
utils.send_mail (mail_message, "")
announce (short_summary, 1)
-#####################################################################################################################
+################################################################################
def reject (changes_filename, manual_reject_mail_filename):
+ global Subst;
+
print "Rejecting.\n"
base_changes_filename = os.path.basename(changes_filename);
# Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored)
try:
utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
- except utils.cant_overwrite_exc:
- sys.stderr.write("W: couldn't overwrite existing file '%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename));
+ except:
+ utils.warn("couldn't reject changes file '%s'. [Got %s]" % (base_changes_filename, sys.exc_type));
pass;
- for file in files.keys():
- if os.path.exists(file):
- try:
- utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
- except utils.cant_overwrite_exc:
- sys.stderr.write("W: couldn't overwrite existing file '%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
- pass;
+ if not changes.has_key("stable install"):
+ for file in files.keys():
+ if os.path.exists(file):
+ try:
+ utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file));
+ except:
+ utils.warn("couldn't reject file '%s'. [Got %s]" % (file, sys.exc_type));
+ pass;
+ else:
+ suite_id = db_access.get_suite_id('proposed-updates');
+ # Remove files from proposed-updates suite
+ for file in files.keys():
+ if files[file]["type"] == "dsc":
+ package = dsc["source"];
+ version = dsc["version"]; # NB: not files[file]["version"], that has no epoch
+ q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version));
+ ql = q.getresult();
+ if not ql:
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version));
+ source_id = ql[0][0];
+ projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id));
+ elif files[file]["type"] == "deb":
+ package = files[file]["package"];
+ version = files[file]["version"];
+ architecture = files[file]["architecture"];
+ q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture));
+ ql = q.getresult();
+ if not ql:
+ utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture));
+ binary_id = ql[0][0];
+ projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id));
# If this is not a manual rejection generate the .reason file and rejection mail message
if manual_reject_mail_filename == "":
fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
os.write(fd, reject_message);
os.close(fd);
- reject_mail_message = """From: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: %s REJECTED
-
-%s
-===
-%s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, reject_footer);
+ Subst["__MANUAL_REJECT_MESSAGE__"] = "";
+ Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)";
+ reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.rejected","r").read());
else: # Have a manual rejection file to use
reject_mail_message = ""; # avoid <undef>'s
-
+
# Send the rejection mail if appropriate
- if not Cnf["Dinstall::Options::No-Mail"]:
+ if not Options["No-Mail"]:
utils.send_mail (reject_mail_message, manual_reject_mail_filename);
+ Logger.log(["rejected", changes_filename]);
+
##################################################################
def manual_reject (changes_filename):
- # Build up the rejection email
- user_email_address = string.replace(string.split(pwd.getpwuid(os.getuid())[4],',')[0], '.', '')
- user_email_address = user_email_address + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
- manual_reject_message = Cnf.get("Dinstall::Options::Manual-Reject", "")
-
- reject_mail_message = """From: %s
-Cc: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: %s REJECTED
-
-%s
-%s
-===
-%s""" % (user_email_address, Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), manual_reject_message, reject_message, reject_footer)
-
+ global Subst;
+
+ # Build up the rejection email
+ user_email_address = utils.whoami() + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"])
+ manual_reject_message = Options.get("Manual-Reject", "")
+
+ Subst["__MANUAL_REJECT_MESSAGE__"] = manual_reject_message;
+ Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"];
+ if changes.has_key("stable install"):
+ template = "katie.stable-rejected";
+ else:
+ template = "katie.rejected";
+ reject_mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/"+template,"r").read());
+
# Write the rejection email out as the <foo>.reason file
reason_filename = re_changes.sub("reason", os.path.basename(changes_filename));
reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename)
fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644);
os.write(fd, reject_mail_message);
os.close(fd);
-
+
# If we weren't given one, spawn an editor so the user can add one in
if manual_reject_message == "":
- result = os.system("vi +6 %s" % (reject_file))
+ result = os.system("vi +6 %s" % (reject_filename))
if result != 0:
- sys.stderr.write ("vi invocation failed for `%s'!" % (reject_file))
- sys.exit(result)
+ utils.fubar("vi invocation failed for `%s'!" % (reject_filename), result);
# Then process it as if it were an automatic rejection
reject (changes_filename, reject_filename)
#####################################################################################################################
-
+
def acknowledge_new (changes_filename, summary):
- global new_ack_new;
+ global new_ack_new, Subst;
changes_filename = os.path.basename(changes_filename);
+ Logger.log(["new",changes_filename]);
+
new_ack_new[changes_filename] = 1;
if new_ack_old.has_key(changes_filename):
return;
print "Sending new ack.";
- if not Cnf["Dinstall::Options::No-Mail"]:
- new_ack_message = """Return-Path: %s
-From: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: %s is NEW
-
-%s
-%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, summary, new_ack_footer);
+ if not Options["No-Mail"]:
+ Subst["__SUMMARY__"] = summary;
+ new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read());
utils.send_mail(new_ack_message,"");
#####################################################################################################################
def announce (short_summary, action):
+ global Subst;
+
# Only do announcements for source uploads with a recent dpkg-dev installed
if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"):
return ""
lists_done = {}
summary = ""
+ Subst["__SHORT_SUMMARY__"] = short_summary;
for dist in changes["distribution"].keys():
list = Cnf.Find("Suite::%s::Announce" % (dist))
- if list == None or lists_done.has_key(list):
+ if list == "" or lists_done.has_key(list):
continue
lists_done[list] = 1
summary = summary + "Announcing to %s\n" % (list)
if action:
- mail_message = """Return-Path: %s
-From: %s
-To: %s
-Bcc: troup@auric.debian.org
-Subject: Installed %s %s (%s)
-
-%s
-
-Installed:
-%s
-""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], list, changes["source"], changes["version"], string.join(changes["architecture"].keys(), ' ' ),
- changes["filecontents"], short_summary)
+ Subst["__ANNOUNCE_LIST_ADDRESS__"] = list;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.announce","r").read());
utils.send_mail (mail_message, "")
- (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]));
bugs = changes["closes"].keys()
bugs.sort()
- if dsc_name == changes["maintainername"]:
+ if not nmu.is_an_nmu(changes, dsc):
summary = summary + "Closing bugs: "
for bug in bugs:
summary = summary + "%s " % (bug)
if action:
- mail_message = """Return-Path: %s
-From: %s
-To: %s-close@bugs.debian.org
-Bcc: troup@auric.debian.org
-Subject: Bug#%s: fixed in %s %s
-
-We believe that the bug you reported is fixed in the latest version of
-%s, which has been installed in the Debian FTP archive:
-
-%s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], bug, bug, changes["source"], changes["version"], changes["source"], short_summary)
-
+ Subst["__BUG_NUMBER__"] = bug;
if changes["distribution"].has_key("stable"):
- mail_message = mail_message + """Note that this package is not part of the released stable Debian
+ Subst["__STABLE_WARNING__"] = """
+Note that this package is not part of the released stable Debian
distribution. It may have dependencies on other unreleased software,
or other instabilities. Please take care if you wish to install it.
The update will eventually make its way into the next released Debian
distribution."""
-
- mail_message = mail_message + """A summary of the changes between this version and the previous one is
-attached.
-
-Thank you for reporting the bug, which will now be closed. If you
-have further comments please address them to %s@bugs.debian.org,
-and the maintainer will reopen the bug report if appropriate.
-
-Debian distribution maintenance software
-pp.
-%s (supplier of updated %s package)
-
-(This message was generated automatically at their request; if you
-believe that there is a problem with it please contact the archive
-administrators by mailing ftpmaster@debian.org)
-
-
-%s""" % (bug, changes["maintainer"], changes["source"], changes["filecontents"])
-
+ else:
+ Subst["__STABLE_WARNING__"] = "";
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read());
utils.send_mail (mail_message, "")
+ if action:
+ Logger.log(["closing bugs"]+bugs);
else: # NMU
summary = summary + "Setting bugs to severity fixed: "
control_message = ""
for bug in bugs:
summary = summary + "%s " % (bug)
- control_message = control_message + "severity %s fixed\n" % (bug)
+ control_message = control_message + "tag %s + fixed\n" % (bug)
if action and control_message != "":
- mail_message = """Return-Path: %s
-From: %s
-To: control@bugs.debian.org
-Bcc: troup@auric.debian.org, %s
-Subject: Fixed in NMU of %s %s
-
-%s
-quit
-
-This message was generated automatically in response to a
-non-maintainer upload. The .changes file follows.
-
-%s
-""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes["maintainer822"], changes["source"], changes["version"], control_message, changes["filecontents"])
+ Subst["__CONTROL_MESSAGE__"] = control_message;
+ mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read());
utils.send_mail (mail_message, "")
+ if action:
+ Logger.log(["setting bugs to fixed"]+bugs);
summary = summary + "\n"
return summary
# into the .changes structure and reprocess the .changes file.
def process_it (changes_file):
- global reprocess, orig_tar_id, changes, dsc, dsc_files, files;
+ global reprocess, orig_tar_id, orig_tar_location, changes, dsc, dsc_files, files, reject_message;
- reprocess = 1;
- orig_tar_id = None;
# Reset some globals
+ reprocess = 1;
changes = {};
dsc = {};
dsc_files = {};
files = {};
orig_tar_id = None;
+ orig_tar_location = "";
legacy_source_untouchable = {};
+ reject_message = "";
# Absolutize the filename to avoid the requirement of being in the
# same directory as the .changes file.
# And since handling of installs to stable munges with the CWD;
# save and restore it.
cwd = os.getcwd();
-
- check_signature (changes_file);
- check_changes (changes_file);
- while reprocess:
- reprocess = 0;
- check_files ();
- check_md5sums ();
- check_dsc ();
- check_diff ();
-
+
+ try:
+ check_signature (changes_file);
+ check_changes (changes_file);
+ while reprocess:
+ reprocess = 0;
+ check_files ();
+ check_md5sums ();
+ check_dsc ();
+ check_diff ();
+ except:
+ print "ERROR";
+ traceback.print_exc(file=sys.stdout);
+ pass;
+
+ update_subst(changes_file);
action(changes_file);
# Restore CWD
###############################################################################
def main():
- global Cnf, projectB, reject_message, install_bytes, new_ack_old
+ global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger
- apt_pkg.init();
-
- Cnf = apt_pkg.newConfiguration();
- apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file());
+ changes_files = init();
- Arguments = [('a',"automatic","Dinstall::Options::Automatic"),
- ('d',"debug","Dinstall::Options::Debug", "IntVal"),
- ('h',"help","Dinstall::Options::Help"),
- ('k',"ack-new","Dinstall::Options::Ack-New"),
- ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"),
- ('n',"no-action","Dinstall::Options::No-Action"),
- ('p',"no-lock", "Dinstall::Options::No-Lock"),
- ('r',"no-version-check", "Dinstall::Options::No-Version-Check"),
- ('s',"no-mail", "Dinstall::Options::No-Mail"),
- ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"),
- ('v',"version","Dinstall::Options::Version")];
-
- changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv);
+ if Options["Help"]:
+ usage();
- if Cnf["Dinstall::Options::Help"]:
- usage(0);
-
- if Cnf["Dinstall::Options::Version"]:
- print "katie version 0.0000000000";
- usage(0);
-
- postgresql_user = None; # Default == Connect as user running program.
+ if Options["Version"]:
+ print "katie %s" % (katie_version);
+ sys.exit(0);
# -n/--dry-run invalidates some other options which would involve things happening
- if Cnf["Dinstall::Options::No-Action"]:
- Cnf["Dinstall::Options::Automatic"] = ""
- Cnf["Dinstall::Options::Ack-New"] = ""
- postgresql_user = Cnf["DB::ROUser"];
+ if Options["No-Action"]:
+ Options["Automatic"] = "";
+ Options["Ack-New"] = "";
- projectB = pg.connect('projectb', Cnf["DB::Host"], int(Cnf["DB::Port"]), None, None, postgresql_user);
+ projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]));
db_access.init(Cnf, projectB);
# Check that we aren't going to clash with the daily cron job
- if os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Cnf["Dinstall::Options::No-Lock"]:
- sys.stderr.write("Archive maintenance in progress. Try again later.\n");
- sys.exit(2);
-
- # Obtain lock if not in no-action mode
+ if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]:
+ utils.fubar("Archive maintenance in progress. Try again later.");
+
+ # Obtain lock if not in no-action mode and initialize the log
- if not Cnf["Dinstall::Options::No-Action"]:
- lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR);
+ if not Options["No-Action"]:
+ lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT);
fcntl.lockf(lock_fd, FCNTL.F_TLOCK);
+ Logger = logging.Logger(Cnf, "katie");
+
+ if Options["Ack-New"]:
+ # Read in the list of already-acknowledged NEW packages
+ if not os.path.exists(Cnf["Dinstall::NewAckList"]):
+ utils.touch_file(Cnf["Dinstall::NewAckList"]);
+ new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"]);
+ new_ack_old = {};
+ for line in new_ack_list.readlines():
+ new_ack_old[line[:-1]] = 1;
+ new_ack_list.close();
+
+ # Initialize the substitution template mapping global
+ Subst = {}
+ Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"];
+ Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"];
+ bcc = "X-Katie: %s" % (katie_version);
+ if Cnf.has_key("Dinstall::Bcc"):
+ Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]);
+ else:
+ Subst["__BCC__"] = bcc;
+ Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"];
+ Subst["__KATIE_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"];
+ Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"];
- # Read in the list of already-acknowledged NEW packages
- new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'r');
- new_ack_old = {};
- for line in new_ack_list.readlines():
- new_ack_old[line[:-1]] = 1;
- new_ack_list.close();
+ # Read in the group-maint override file
+ nmu = nmu_p();
+
+ # Sort the .changes files so that we process sourceful ones first
+ changes_files.sort(utils.changes_compare);
# Process the changes files
for changes_file in changes_files:
- reject_message = ""
print "\n" + changes_file;
process_it (changes_file);
if install_count > 1:
sets = "sets"
sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes))));
+ Logger.log(["total",install_count,install_bytes]);
# Write out the list of already-acknowledged NEW packages
- if Cnf["Dinstall::Options::Ack-New"]:
+ if Options["Ack-New"]:
new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w')
for i in new_ack_new.keys():
new_ack_list.write(i+'\n')
new_ack_list.close()
-
-
+
+ if not Options["No-Action"]:
+ Logger.close();
+
if __name__ == '__main__':
main()
projects / dak.git / blobdiff