X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=katie;h=cf26804343ea59c541b53fb202d2016e36a89a6b;hb=6d72ae4bca15c0ed410798e62d3e367cf7abcdb6;hp=700791ca33f6d813d6dff775070fa3d59542363e;hpb=6ada259d555e36a1546789c280edc3f944b7de7f;p=dak.git diff --git a/katie b/katie index 700791ca..cf268043 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000, 2001 James Troup -# $Id: katie,v 1.47 2001-06-22 22:53:14 troup Exp $ +# $Id: katie,v 1.56 2001-07-28 18:07:58 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,9 @@ import FCNTL, commands, fcntl, getopt, gzip, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time, traceback import apt_inst, apt_pkg -import utils, db_access +import utils, db_access, logging + +from types import *; ############################################################################### @@ -43,12 +45,15 @@ re_changes = re.compile (r"changes$"); re_default_answer = re.compile(r"\[(.*)\]"); re_fdnic = re.compile("\n\n"); re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); -re_bin_only_nmu = re.compile("\.\d+\.\d+$"); +re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$"); +re_bin_only_nmu_of_nmu = re.compile("\.\d+$"); ######################################################################################### # Globals Cnf = None; +Options = None; +Logger = None; reject_message = ""; changes = {}; dsc = {}; @@ -88,7 +93,7 @@ def check_signature (filename): (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename)) if (result != 0): - reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output) + reject_message = reject_message + "Rejected: GPG signature check failed on `%s'.\n%s\n" % (os.path.basename(filename), output) return 0 return 1 @@ -110,16 +115,16 @@ class nmu_p: def is_an_nmu (self, changes, dsc): (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])); # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry - if dsc_name == changes["maintainername"] and (changes["changedbyname"] == "" or changes["changedbyname"] == dsc_name): + if dsc_name == changes["maintainername"] and (changes["changedby822"] == "" or changes["changedbyname"] == dsc_name): return 0; - if dsc.has_key("maintainers"): - maintainers = string.split(dsc["maintainers"], ","); - maintainernames = {}; - for i in maintainers: + if dsc.has_key("uploaders"): + uploaders = string.split(dsc["uploaders"], ","); + uploadernames = {}; + for i in uploaders: (rfc822, name, email) = utils.fix_maintainer (string.strip(i)); - maintainernames[name] = ""; - if maintainernames.has_key(changes["changedbyname"]): + uploadernames[name] = ""; + if uploadernames.has_key(changes["changedbyname"]): return 0; # Some group maintained packages (e.g. Debian QA) are never NMU's @@ -130,6 +135,38 @@ class nmu_p: ###################################################################################################### +# Ensure that source exists somewhere in the archive for the binary +# upload being processed. +# +# (1) exact match => 1.0-3 +# (2) Bin-only NMU of an MU => 1.0-3.0.1 +# (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1 + +def source_exists (package, source_version): + q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (package)); + + # Reduce the query results to a list of version numbers + ql = map(lambda x: x[0], q.getresult()); + + # Try (1) + if ql.count(source_version): + return 1; + + # Try (2) + orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version); + if ql.count(orig_source_version): + return 1; + + # Try (3) + orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version); + if ql.count(orig_source_version): + return 1; + + # No source found... + return 0; + +###################################################################################################### + # See if a given package is in the override table def in_override_p (package, component, suite, binary_type, file): @@ -186,28 +223,31 @@ def check_changes(filename): try: changes = utils.parse_changes(filename, 0) except utils.cant_open_exc: - reject_message = "Rejected: can't read changes file '%s'.\n" % (filename) + reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename) return 0; except utils.changes_parse_error_exc, line: - reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) + reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) return 0; # Parse the Files field from the .changes into another dictionary try: files = utils.build_file_list(changes, ""); except utils.changes_parse_error_exc, line: - reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line); + reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line); + except utils.nk_format_exc, format: + reject_message = reject_message + "Rejected: unknown format '%s' of changes file '%s'.\n" % (format, filename); + return 0; # Check for mandatory fields - for i in ("source", "binary", "architecture", "version", "distribution","maintainer", "files"): + for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"): if not changes.has_key(i): - reject_message = "Rejected: Missing field `%s' in changes file.\n" % (i) + reject_message = reject_message + "Rejected: Missing field `%s' in changes file.\n" % (i) return 0 # Avoid errors during later tests # Override the Distribution: field if appropriate - if Cnf["Dinstall::Options::Override-Distribution"] != "": - reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"]) - changes["distribution"] = Cnf["Dinstall::Options::Override-Distribution"] + if Options["Override-Distribution"] != "": + reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Options["Override-Distribution"]) + changes["distribution"] = Options["Override-Distribution"] # Split multi-value fields into a lower-level dictionary for i in ("architecture", "distribution", "binary", "closes"): @@ -230,6 +270,10 @@ def check_changes(filename): if re_isanum.match (i) == None: reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i) + # Ensure there _is_ a target distribution + if changes["distribution"].keys() == []: + reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n"; + # Map frozen to unstable if frozen doesn't exist if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"): del changes["distribution"]["frozen"] @@ -238,19 +282,17 @@ def check_changes(filename): # Map testing to unstable if changes["distribution"].has_key("testing"): - del changes["distribution"]["testing"] - changes["distribution"]["unstable"] = 1; - reject_message = reject_message + "Mapping testing to unstable.\n" + if len(changes["distribution"].keys()) > 1: + del changes["distribution"]["testing"]; + reject_message = reject_message + "Warning: Ignoring testing as a target suite.\n"; + else: + reject_message = reject_message + "Rejected: invalid distribution 'testing'.\n"; # Ensure target distributions exist for i in changes["distribution"].keys(): if not Cnf.has_key("Suite::%s" % (i)): reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i) - # Ensure there _is_ a target distribution - if changes["distribution"].keys() == []: - reject_message = reject_message + "Rejected: huh? Distribution field is empty in changes file.\n"; - # Map unreleased arches from stable to unstable if changes["distribution"].has_key("stable"): for i in changes["architecture"].keys(): @@ -388,7 +430,7 @@ def check_files(): files[file]["package"] = m.group(1) files[file]["version"] = m.group(2) files[file]["type"] = m.group(3) - + # Ensure the source package name matches the Source filed in the .changes if changes["source"] != files[file]["package"]: reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"]) @@ -410,6 +452,7 @@ def check_files(): check_signature(file) files[file]["fullname"] = file + files[file]["architecture"] = "source"; # Not a binary or source package? Assume byhand... else: @@ -432,7 +475,7 @@ def check_files(): if files[file]["type"] == "deb": # Find any old binary packages - q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND a.arch_string = '%s' AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id" + q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id" % (files[file]["package"], suite, files[file]["architecture"])) oldfiles = q.dictresult() for oldfile in oldfiles: @@ -451,18 +494,10 @@ def check_files(): if changes["architecture"].has_key("source"): source_version = files[file]["source version"]; if source_version != changes["version"]: - reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["sourceversion"], file, changes["version"]); + reject_message = reject_message + "Rejected: source version (%s) for %s doesn't match changes version %s.\n" % (files[file]["source version"], file, changes["version"]); else: - q = projectB.query("SELECT s.version FROM source s WHERE s.source = '%s'" % (files[file]["source package"])); - ql = map(lambda x: x[0], q.getresult()); - if ql.count(source_version) == 0: - # Maybe it's a binary only NMU ? - if re_bin_only_nmu.search(source_version): - orig_source_version = re_bin_only_nmu.sub('', source_version); - if ql.count(orig_source_version) == 0: - reject_message = reject_message + "Rejected: no source version (%s [or %s]) found in %s for %s (%s).\n" % (source_version, orig_source_version, suite, files[file]["source package"], file); - else: - reject_message = reject_message + "Rejected: no source version (%s) found in %s for %s (%s).\n" % (source_version, suite, files[file]["source package"], file); + if not source_exists (files[file]["source package"], source_version): + reject_message = reject_message + "Rejected: no source found for %s %s (%s).\n" % (files[file]["source package"], source_version, file); # Find any old .dsc files elif files[file]["type"] == "dsc": @@ -529,9 +564,40 @@ def check_dsc (): reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n"; continue; except utils.changes_parse_error_exc, line: - reject_message = "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line); + reject_message = reject_message + "Rejected: error parsing .dsc file '%s', can't grok: %s.\n" % (file, line); continue; + # Enforce mandatory fields + for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): + if not dsc.has_key(i): + reject_message = reject_message + "Rejected: Missing field `%s' in dsc file.\n" % (i) + + # The dpkg maintainer from hell strikes again! Bumping the + # version number of the .dsc breaks extraction by stable's + # dpkg-source. + if dsc["format"] != "1.0": + reject_message = reject_message + """Rejected: [dpkg-sucks] source package was produced by a broken version + of dpkg-dev 1.9.1{3,4}; please rebuild with >= 1.9.15 version + installed. +"""; + + # Ensure the version number in the .dsc matches the version number in the .changes + epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version")); + changes_version = files[file]["version"]; + if epochless_dsc_version != files[file]["version"]: + reject_message = reject_message + "Rejected: version ('%s') in .dsc does not match version ('%s') in .changes\n" % (epochless_dsc_version, changes_version); + + # Ensure source is newer than existing source in target suites + package = dsc.get("source"); + new_version = dsc.get("version"); + for suite in changes["distribution"].keys(): + q = projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id" + % (package, suite)); + ql = map(lambda x: x[0], q.getresult()); + for old_version in ql: + if apt_pkg.VersionCompare(new_version, old_version) != 1: + reject_message = reject_message + "Rejected: %s Old version `%s' >= new version `%s'.\n" % (file, old_version, new_version) + # Try and find all files mentioned in the .dsc. This has # to work harder to cope with the multiple possible # locations of an .orig.tar.gz. @@ -696,10 +762,14 @@ def check_override (): def update_subst (changes_filename): global Subst; - if changes.has_key("architecture"): - Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' ); - else: - Subst["__ARCHITECTURE__"] = "Unknown"; + # If katie crashed out in the right place, architecture may still be a string. + if not changes.has_key("architecture") or not isinstance(changes["architecture"], DictType): + changes["architecture"] = { "Unknown" : "" }; + # and maintainer822 may not exist. + if not changes.has_key("maintainer822"): + changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"]; + + Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' ); Subst["__CHANGES_FILENAME__"] = os.path.basename(changes_filename); Subst["__FILE_CONTENTS__"] = changes.get("filecontents", ""); @@ -724,7 +794,7 @@ def action (changes_filename): # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) - if not changes.has_key("distribution"): + if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType): changes["distribution"] = {}; for suite in changes["distribution"].keys(): @@ -761,7 +831,7 @@ def action (changes_filename): summary = summary + announce (short_summary, 0) (prompt, answer) = ("", "XXX") - if Cnf["Dinstall::Options::No-Action"] or Cnf["Dinstall::Options::Automatic"]: + if Options["No-Action"] or Options["Automatic"]: answer = 'S' if string.find(reject_message, "Rejected") != -1: @@ -775,12 +845,12 @@ def action (changes_filename): else: print "REJECT\n" + reject_message,; prompt = "[R]eject, Manual reject, Skip, Quit ?"; - if Cnf["Dinstall::Options::Automatic"]: + if Options["Automatic"]: answer = 'R'; elif new: print "NEW to %s\n%s%s" % (suites, reject_message, summary),; prompt = "[S]kip, New ack, Manual reject, Quit ?"; - if Cnf["Dinstall::Options::Automatic"] and Cnf["Dinstall::Options::Ack-New"]: + if Options["Automatic"] and Options["Ack-New"]: answer = 'N'; elif byhand: print "BYHAND\n" + reject_message + summary,; @@ -791,7 +861,7 @@ def action (changes_filename): else: print "INSTALL\n" + reject_message + summary,; prompt = "[I]nstall, Manual reject, Skip, Quit ?"; - if Cnf["Dinstall::Options::Automatic"]: + if Options["Automatic"]: answer = 'I'; while string.find(prompt, answer) == -1: @@ -825,6 +895,8 @@ def install (changes_filename, summary, short_summary): print "Installing." + Logger.log(["installing changes",changes_filename]); + archive = utils.where_am_i(); # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. @@ -844,7 +916,7 @@ def install (changes_filename, summary, short_summary): files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id) projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)" % (package, version, maintainer_id, files[file]["files id"])) - + for suite in changes["distribution"].keys(): suite_id = db_access.get_suite_id(suite); projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id)) @@ -935,6 +1007,7 @@ def install (changes_filename, summary, short_summary): destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file destdir = os.path.dirname(destination) utils.move (file, destination) + Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]); install_bytes = install_bytes + float(files[file]["size"]) # Copy the .changes file across for suite which need it. @@ -951,7 +1024,7 @@ def install (changes_filename, summary, short_summary): install_count = install_count + 1; - if not Cnf["Dinstall::Options::No-Mail"]: + if not Options["No-Mail"]: Subst["__SUITE__"] = ""; Subst["__SUMMARY__"] = summary; mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read()); @@ -1034,7 +1107,7 @@ def stable_install (changes_filename, summary, short_summary): install_count = install_count + 1; - if not Cnf["Dinstall::Options::No-Mail"]: + if not Options["No-Mail"]: Subst["__SUITE__"] = " into stable"; Subst["__SUMMARY__"] = summary; mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read()); @@ -1080,9 +1153,11 @@ def reject (changes_filename, manual_reject_mail_filename): reject_mail_message = ""; # avoid 's # Send the rejection mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: + if not Options["No-Mail"]: utils.send_mail (reject_mail_message, manual_reject_mail_filename); + Logger.log(["rejected", changes_filename]); + ################################################################## def manual_reject (changes_filename): @@ -1128,7 +1203,7 @@ def acknowledge_new (changes_filename, summary): return; print "Sending new ack."; - if not Cnf["Dinstall::Options::No-Mail"]: + if not Options["No-Mail"]: Subst["__SUMMARY__"] = summary; new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.new","r").read()); utils.send_mail(new_ack_message,""); @@ -1177,6 +1252,8 @@ distribution.""" Subst["__STABLE_WARNING__"] = ""; mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-close","r").read()); utils.send_mail (mail_message, "") + if action: + Logger.log(["closing bugs"]+bugs); else: # NMU summary = summary + "Setting bugs to severity fixed: " control_message = "" @@ -1187,6 +1264,8 @@ distribution.""" Subst["__CONTROL_MESSAGE__"] = control_message; mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.bug-nmu-fixed","r").read()); utils.send_mail (mail_message, "") + if action: + Logger.log(["setting bugs to fixed"]+bugs); summary = summary + "\n" return summary @@ -1246,7 +1325,7 @@ def process_it (changes_file): ############################################################################### def main(): - global Cnf, projectB, install_bytes, new_ack_old, Subst, nmu + global Cnf, Options, projectB, install_bytes, new_ack_old, Subst, nmu, Logger apt_pkg.init(); @@ -1265,20 +1344,21 @@ def main(): ('v',"version","Dinstall::Options::Version")]; changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + Options = Cnf.SubTree("Dinstall::Options") - if Cnf["Dinstall::Options::Help"]: + if Options["Help"]: usage(0); - if Cnf["Dinstall::Options::Version"]: + if Options["Version"]: print "katie version 0.0000000000"; usage(0); postgresql_user = None; # Default == Connect as user running program. # -n/--dry-run invalidates some other options which would involve things happening - if Cnf["Dinstall::Options::No-Action"]: - Cnf["Dinstall::Options::Automatic"] = "" - Cnf["Dinstall::Options::Ack-New"] = "" + if Options["No-Action"]: + Options["Automatic"] = ""; + Options["Ack-New"] = ""; postgresql_user = Cnf["DB::ROUser"]; projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"]), None, None, postgresql_user); @@ -1287,14 +1367,15 @@ def main(): # Check that we aren't going to clash with the daily cron job - if os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Cnf["Dinstall::Options::No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); - # Obtain lock if not in no-action mode + # Obtain lock if not in no-action mode and initialize the log - if not Cnf["Dinstall::Options::No-Action"]: + if not Options["No-Action"]: lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR); fcntl.lockf(lock_fd, FCNTL.F_TLOCK); + Logger = logging.Logger(Cnf, "katie"); # Read in the list of already-acknowledged NEW packages new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'r'); @@ -1307,7 +1388,7 @@ def main(): Subst = {} Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]; Subst["__BUG_SERVER__"] = Cnf["Dinstall::BugServer"]; - bcc = "X-Katie: $Revision: 1.47 $" + bcc = "X-Katie: $Revision: 1.56 $" if Cnf.has_key("Dinstall::Bcc"): Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); else: @@ -1331,14 +1412,17 @@ def main(): if install_count > 1: sets = "sets" sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes)))); + Logger.log(["total",install_count,install_bytes]); # Write out the list of already-acknowledged NEW packages - if Cnf["Dinstall::Options::Ack-New"]: + if Options["Ack-New"]: new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w') for i in new_ack_new.keys(): new_ack_list.write(i+'\n') new_ack_list.close() + if not Options["No-Action"]: + Logger.close(); if __name__ == '__main__': main()