X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=katie;h=27d909b7adf0028ce952ab56ea63deb21046dd6b;hb=30b41506a6105ab23a0cac9e8197475a89236224;hp=010d56119b77bcaf5ff8de823d0676e13a9747fd;hpb=4df20c3923c3b768e31030a436b77829f56776ac;p=dak.git diff --git a/katie b/katie index 010d5611..27d909b7 100755 --- a/katie +++ b/katie @@ -1,8 +1,8 @@ #!/usr/bin/env python -# Installs Debian packaes -# Copyright (C) 2000 James Troup -# $Id: katie,v 1.14 2000-12-19 21:06:20 troup Exp $ +# Installs Debian packages +# Copyright (C) 2000, 2001, 2002 James Troup +# $Id: katie,v 1.80 2002-05-08 11:52:31 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +18,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# Based (almost entirely) on dinstall by Guy Maor +# Originally based on dinstall by Guy Maor -######################################################################################### +############################################################################### # Cartman: "I'm trying to make the best of a bad situation, I don't # need to hear crap from a bunch of hippy freaks living in @@ -30,602 +30,226 @@ # # Cartman: "uhh.. screw you guys... home." -######################################################################################### - -import FCNTL, commands, fcntl, getopt, os, pg, pwd, re, shutil, stat, string, sys, tempfile, time -import apt_inst, apt_pkg -import utils, db_access - ############################################################################### -re_isanum = re.compile (r'^\d+$'); -re_isadeb = re.compile (r'.*\.u?deb$'); -re_issource = re.compile (r'(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)'); -re_dpackage = re.compile (r'^package:\s*(.*)', re.IGNORECASE); -re_darchitecture = re.compile (r'^architecture:\s*(.*)', re.IGNORECASE); -re_dversion = re.compile (r'^version:\s*(.*)', re.IGNORECASE); -re_dsection = re.compile (r'^section:\s*(.*)', re.IGNORECASE); -re_dpriority = re.compile (r'^priority:\s*(.*)', re.IGNORECASE); -re_changes = re.compile (r'changes$'); -re_override_package = re.compile(r'(\S*)\s+.*'); -re_default_answer = re.compile(r"\[(.*)\]"); -re_fdnic = re.compile("\n\n"); +import FCNTL, fcntl, os, string, sys, time; +import apt_pkg; +import db_access, katie, logging, utils; ############################################################################### -# -reject_footer = """If you don't understand why your files were rejected, or if the -override file requires editing, reply to this email. - -Your rejected files are in incoming/REJECT/. (Some may also be in -incoming/ if your .changes file was unparsable.) If only some of the -files need to repaired, you may move any good files back to incoming/. -Please remove any bad files from incoming/REJECT/.""" -# -new_ack_footer = """Your package contains new components which requires manual editing of -the override file. It is ok otherwise, so please be patient. New -packages are usually added to the override file about once a week. - -You may have gotten the distribution wrong. You'll get warnings above -if files already exist in other distributions.""" -# -installed_footer = """If the override file requires editing, file a bug on ftp.debian.org. - -Thank you for your contribution to Debian GNU.""" - -######################################################################################### - # Globals +katie_version = "$Revision: 1.80 $"; + Cnf = None; -reject_message = ""; -changes = {}; -dsc = {}; -dsc_files = {}; -files = {}; +Options = None; +Logger = None; +Urgency_Logger = None; projectB = None; -new_ack_new = {}; -new_ack_old = {}; -overrides = {}; +Katie = None; +pkg = None; + +reject_message = ""; +changes = None; +dsc = None; +dsc_files = None; +files = None; +Subst = None; + install_count = 0; install_bytes = 0.0; -reprocess = 0; -orig_tar_id = None; -######################################################################################### +installing_to_stable = 0; -def usage (exit_code): - print """Usage: dinstall [OPTION]... [CHANGES]... - -a, --automatic automatic run - -d, --debug=VALUE debug - -k, --ack-new acknowledge new packages - -m, --manual-reject=MSG manual reject with `msg' - -n, --dry-run don't do anything - -p, --no-lock don't check lockfile !! for cron.daily only !! - -r, --no-version-check override version check - -u, --distribution=DIST override distribution to `dist'""" - sys.exit(exit_code) - -def check_signature (filename): - global reject_message - - (result, output) = commands.getstatusoutput("gpg --emulate-md-encode-bug --batch --no-options --no-default-keyring --always-trust --keyring=%s --keyring=%s < %s >/dev/null" % (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"], filename)) - if (result != 0): - reject_message = "Rejected: GPG signature check failed on `%s'.\n%s\n" % (filename, output) - return 0 - return 1 - -##################################################################################################################### - -def read_override_file (filename, suite, component, binary_type): - global overrides; - - file = utils.open_file(filename, 'r'); - for line in file.readlines(): - line = string.strip(utils.re_comments.sub('', line)) - override_package = re_override_package.sub(r'\1', line) - if override_package != "": - overrides[suite][component][binary_type][override_package] = 1 - file.close() - - -# See if a given package is in the override file. Caches and only loads override files on demand. - -def in_override_p (package, component, suite, binary_type): - global overrides; - - if binary_type == "" or binary_type == "deb": - binary_type = "-"; - - # Avoid on unknown distributions - if db_access.get_suite_id(suite) == -1: - return None; - - # FIXME: nasty non-US speficic hack - if string.lower(component[:7]) == "non-us/": - component = component[7:]; - if not overrides.has_key(suite) or not overrides[suite].has_key(component) or not overrides[suite][component].has_key(binary_type): - if not overrides.has_key(suite): - overrides[suite] = {} - if not overrides[suite].has_key(component): - overrides[suite][component] = {} - if not overrides[suite][component].has_key(binary_type): - overrides[suite][component][binary_type] = {} - if Cnf.has_key("Suite::%s::SingleOverrideFile" % (suite)): # legacy mixed suite (i.e. experimental) - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)]; - read_override_file (override_filename, suite, component, binary_type); - else: # all others. - if binary_type == "udeb": - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.debian-installer.' + component; - read_override_file (override_filename, suite, component, binary_type); - else: - for src in ("", ".src"): - override_filename = Cnf["Dir::OverrideDir"] + 'override.' + Cnf["Suite::%s::OverrideCodeName" % (suite)] + '.' + component + src; - read_override_file (override_filename, suite, component, binary_type); +############################################################################### - return overrides[suite][component][binary_type].get(package, None); +# FIXME: this should go away to some Debian specific file +# FIXME: should die if file already exists + +class Urgency_Log: + "Urgency Logger object" + def __init__ (self, Cnf): + "Initialize a new Urgency Logger object" + self.Cnf = Cnf; + self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())); + # Create the log directory if it doesn't exist + self.log_dir = Cnf["Dir::UrgencyLog"]; + if not os.path.exists(self.log_dir): + umask = os.umask(00000); + os.makedirs(self.log_dir, 02775); + # Open the logfile + self.log_filename = "%s/.install-urgencies-%s.new" % (self.log_dir, self.timestamp); + self.log_file = utils.open_file(self.log_filename, 'w'); + self.writes = 0; + + def log (self, source, version, urgency): + "Log an event" + self.log_file.write(string.join([source, version, urgency])+'\n'); + self.log_file.flush(); + self.writes = self.writes + 1; + + def close (self): + "Close a Logger object" + self.log_file.flush(); + self.log_file.close(); + if self.writes: + new_filename = "%s/install-urgencies-%s" % (self.log_dir, self.timestamp); + utils.move(self.log_filename, new_filename); + else: + os.unlink(self.log_filename); -##################################################################################################################### +############################################################################### -def check_changes(filename): - global reject_message, changes, files +def reject (str, prefix="Rejected: "): + global reject_message; + if str: + reject_message = reject_message + prefix + str + "\n"; - # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.] - try: - changes = utils.parse_changes(filename) - except utils.cant_open_exc: - reject_message = "Rejected: can't read changes file '%s'.\n" % (filename) - return 0; - except utils.changes_parse_error_exc, line: - reject_message = "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) - changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"]; - return 0; - - # Parse the Files field from the .changes into another dictionary [FIXME need to trap errors as above] - files = utils.build_file_list(changes, "") - - # Check for mandatory fields - for i in ("source", "binary", "architecture", "version", "distribution","maintainer", "files"): - if not changes.has_key(i): - reject_message = "Rejected: Missing field `%s' in changes file." % (i) - return 0 # Avoid errors during later tests - - # Fix the Maintainer: field to be RFC822 compatible - (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) - - # Override the Distribution: field if appropriate - if Cnf["Dinstall::Options::Override-Distribution"] != "": - reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"]) - changes["distribution"] = Cnf["Dinstall::Options::Override-Distribution"] - - # Split multi-value fields into a lower-level dictionary - for i in ("architecture", "distribution", "binary", "closes"): - o = changes.get(i, "") - if o != "": - del changes[i] - changes[i] = {} - for j in string.split(o): - changes[i][j] = 1 - - # Ensure all the values in Closes: are numbers - if changes.has_key("closes"): - for i in changes["closes"].keys(): - if re_isanum.match (i) == None: - reject_message = reject_message + "Rejected: `%s' from Closes field isn't a number.\n" % (i) - - # Map frozen to unstable if frozen doesn't exist - if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"): - del changes["distribution"]["frozen"] - reject_message = reject_message + "Mapping frozen to unstable.\n" - - # Ensure target distributions exist - for i in changes["distribution"].keys(): - if not Cnf.has_key("Suite::%s" % (i)): - reject_message = reject_message + "Rejected: Unknown distribution `%s'.\n" % (i) - - # Map unreleased arches from stable to unstable - if changes["distribution"].has_key("stable"): - for i in changes["architecture"].keys(): - if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)): - reject_message = reject_message + "Mapping stable to unstable for unreleased arch `%s'.\n" % (i) - del changes["distribution"]["stable"] - - # Map arches not being released from frozen to unstable - if changes["distribution"].has_key("frozen"): - for i in changes["architecture"].keys(): - if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)): - reject_message = reject_message + "Mapping frozen to unstable for non-releasing arch `%s'.\n" % (i) - del changes["distribution"]["frozen"] - - # Handle uploads to stable - if changes["distribution"].has_key("stable"): - # If running from within proposed-updates; assume an install to stable - if string.find(os.getcwd(), 'proposed-updates') != -1: - # FIXME: should probably remove anything that != stable - for i in ("frozen", "unstable"): - if changes["distribution"].has_key(i): - reject_message = reject_message + "Removing %s from distribution list.\n" - del changes["distribution"][i] - changes["stable upload"] = 1; - # If we can't find a file from the .changes; assume it's a package already in the pool and move into the pool - file = files.keys()[0]; - if os.access(file, os.R_OK) == 0: - pool_dir = Cnf["Dir::PoolDir"] + '/' + utils.poolify(changes["source"], files[file]["component"]); - os.chdir(pool_dir); - # Otherwise (normal case) map stable to updates - else: - reject_message = reject_message + "Mapping stable to updates.\n"; - del changes["distribution"]["stable"]; - changes["distribution"]["proposed-updates"] = 1; - - # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison) - changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"]) - changes["chopversion2"] = utils.re_no_revision.sub('', changes["chopversion"]) - - if string.find(reject_message, "Rejected:") != -1: - return 0 - else: - return 1 - -def check_files(): - global reject_message - - archive = utils.where_am_i(); +# Recheck anything that relies on the database; since that's not +# frozen between accept and katie's run time. +def check(): for file in files.keys(): - # Check the file is readable - if os.access(file,os.R_OK) == 0: - reject_message = reject_message + "Rejected: Can't read `%s'.\n" % (file) - files[file]["type"] = "unreadable"; - continue - # If it's byhand skip remaining checks - if files[file]["section"] == "byhand": - files[file]["byhand"] = 1; - files[file]["type"] = "byhand"; - # Checks for a binary package... - elif re_isadeb.match(file) != None: - # Extract package information using dpkg-deb - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r"))) - - # Check for mandatory fields - if control.Find("Package") == None: - reject_message = reject_message + "Rejected: %s: No package field in control.\n" % (file) - if control.Find("Architecture") == None: - reject_message = reject_message + "Rejected: %s: No architecture field in control.\n" % (file) - if control.Find("Version") == None: - reject_message = reject_message + "Rejected: %s: No version field in control.\n" % (file) - - # Ensure the package name matches the one give in the .changes - if not changes["binary"].has_key(control.Find("Package", "")): - reject_message = reject_message + "Rejected: %s: control file lists name as `%s', which isn't in changes file.\n" % (file, control.Find("Package", "")) - - # Validate the architecture - if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))): - reject_message = reject_message + "Rejected: Unknown architecture '%s'.\n" % (control.Find("Architecture", "")) - - # Check the architecture matches the one given in the .changes - if not changes["architecture"].has_key(control.Find("Architecture", "")): - reject_message = reject_message + "Rejected: %s: control file lists arch as `%s', which isn't in changes file.\n" % (file, control.Find("Architecture", "")) - # Check the section & priority match those given in the .changes (non-fatal) - if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"): - reject_message = reject_message + "Warning: %s control file lists section as `%s', but changes file has `%s'.\n" % (file, control.Find("Section", ""), files[file]["section"]) - if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"): - reject_message = reject_message + "Warning: %s control file lists priority as `%s', but changes file has `%s'.\n" % (file, control.Find("Priority", ""), files[file]["priority"]) - - epochless_version = utils.re_no_epoch.sub('', control.Find("Version", "")) - - files[file]["package"] = control.Find("Package"); - files[file]["architecture"] = control.Find("Architecture"); - files[file]["version"] = control.Find("Version"); - files[file]["maintainer"] = control.Find("Maintainer", ""); - if file[-5:] == ".udeb": - files[file]["dbtype"] = "udeb"; - elif file[-4:] == ".deb": - files[file]["dbtype"] = "deb"; - else: - reject_message = reject_message + "Rejected: %s is neither a .deb or a .udeb.\n " % (file); - files[file]["type"] = "deb"; - files[file]["fullname"] = "%s_%s_%s.deb" % (control.Find("Package", ""), epochless_version, control.Find("Architecture", "")) - files[file]["source"] = control.Find("Source", ""); - if files[file]["source"] == "": - files[file]["source"] = files[file]["package"]; - # Checks for a source package... - else: - m = re_issource.match(file) - if m != None: - files[file]["package"] = m.group(1) - files[file]["version"] = m.group(2) - files[file]["type"] = m.group(3) - - # Ensure the source package name matches the Source filed in the .changes - if changes["source"] != files[file]["package"]: - reject_message = reject_message + "Rejected: %s: changes file doesn't say %s for Source\n" % (file, files[file]["package"]) - - # Ensure the source version matches the version in the .changes file - if files[file]["type"] == "orig.tar.gz": - changes_version = changes["chopversion2"] - else: - changes_version = changes["chopversion"] - if changes_version != files[file]["version"]: - reject_message = reject_message + "Rejected: %s: should be %s according to changes file.\n" % (file, changes_version) - - # Ensure the .changes lists source in the Architecture field - if not changes["architecture"].has_key("source"): - reject_message = reject_message + "Rejected: %s: changes file doesn't list `source' in Architecture field.\n" % (file) - - # Check the signature of a .dsc file - if files[file]["type"] == "dsc": - check_signature(file) - - files[file]["fullname"] = file - - # Not a binary or source package? Assume byhand... - else: - files[file]["byhand"] = 1; - files[file]["type"] = "byhand"; - - files[file]["oldfiles"] = {} + # Check that the source still exists + if files[file]["type"] == "deb": + source_version = files[file]["source version"]; + source_package = files[file]["source package"]; + if not changes["architecture"].has_key("source") \ + and not Katie.source_exists(source_package, source_version): + reject("no source found for %s %s (%s)." % (source_package, source_version, file)); + for suite in changes["distribution"].keys(): - # Skip byhand - if files[file].has_key("byhand"): - continue - - if Cnf.has_key("Suite:%s::Components" % (suite)) and not Cnf.has_key("Suite::%s::Components::%s" % (suite, files[file]["component"])): - reject_message = reject_message + "Rejected: unknown component `%s' for suite `%s'.\n" % (files[file]["component"], suite) - continue - - # See if the package is NEW - if not in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype","")): - files[file]["new"] = 1 - - # Find any old binary packages - if files[file]["type"] == "deb": - q = projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND a.arch_string = '%s' AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id" - % (files[file]["package"], suite, files[file]["architecture"])) - oldfiles = q.dictresult() - for oldfile in oldfiles: - files[file]["oldfiles"][suite] = oldfile - # Check versions [NB: per-suite only; no cross-suite checking done (yet)] - if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1: - if Cnf["Dinstall::Options::No-Version-Check"]: - reject_message = reject_message + "Overriden rejection" - else: - reject_message = reject_message + "Rejected" - reject_message = reject_message + ": %s Old version `%s' >= new version `%s'.\n" % (file, oldfile["version"], files[file]["version"]) - # Check for existing copies of the file - if not changes.has_key("stable upload"): - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"])) - if q.getresult() != []: - reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (file) - - # Find any old .dsc files - elif files[file]["type"] == "dsc": - q = projectB.query("SELECT s.id, s.version, f.filename, l.path, c.name FROM source s, src_associations sa, suite su, location l, component c, files f WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id AND f.location = l.id AND l.component = c.id AND f.id = s.file" - % (files[file]["package"], suite)) - oldfiles = q.dictresult() - if len(oldfiles) >= 1: - files[file]["oldfiles"][suite] = oldfiles[0] - - # Validate the component - component = files[file]["component"]; - component_id = db_access.get_component_id(component); - if component_id == -1: - reject_message = reject_message + "Rejected: file '%s' has unknown component '%s'.\n" % (file, component); - continue; - - # Check the md5sum & size against existing files (if any) - location = Cnf["Dir::PoolDir"]; - files[file]["location id"] = db_access.get_location_id (location, component, archive); - - files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]); - files_id = db_access.get_files_id(files[file]["pool name"] + file, files[file]["size"], files[file]["md5sum"], files[file]["location id"]); - if files_id == -1: - reject_message = reject_message + "Rejected: INTERNAL ERROR, get_files_id() returned multiple matches for %s.\n" % (file) - elif files_id == -2: - reject_message = reject_message + "Rejected: md5sum and/or size mismatch on existing copy of %s.\n" % (file) - files[file]["files id"] = files_id - - # Check for packages that have moved from one component to another - if files[file]["oldfiles"].has_key(suite) and files[file]["oldfiles"][suite]["name"] != files[file]["component"]: - files[file]["othercomponents"] = files[file]["oldfiles"][suite]["name"]; - - - if string.find(reject_message, "Rejected:") != -1: - return 0 - else: - return 1 + # Check the package is still in the override tables + if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + reject("%s is NEW for %s." % (file, suite)); + + if not installing_to_stable: + if files[file]["type"] == "deb": + reject(Katie.check_binaries_against_db(file, suite)); + elif files[file]["type"] == "dsc": + reject(Katie.check_source_against_db(file)); + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); + reject(reject_msg); ############################################################################### -def check_dsc (): - global dsc, dsc_files, reject_message, reprocess, orig_tar_id; - - for file in files.keys(): - if files[file]["type"] == "dsc": - try: - dsc = utils.parse_changes(file) - except utils.cant_open_exc: - reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename) - return 0; - except utils.changes_parse_error_exc, line: - reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) - return 0; - try: - dsc_files = utils.build_file_list(dsc, 1) - except utils.no_files_exc: - reject_message = reject_message + "Rejected: no Files: field in .dsc file.\n"; - continue; +def init(): + global Cnf, Options, Katie, projectB, changes, dsc, dsc_files, files, pkg, Subst; - # Try and find all files mentioned in the .dsc. This has - # to work harder to cope with the multiple possible - # locations of an .orig.tar.gz. - for dsc_file in dsc_files.keys(): - if files.has_key(dsc_file): - actual_md5 = files[dsc_file]["md5sum"] - found = "%s in incoming" % (dsc_file) - # Check the file does not already exist in the archive - if not changes.has_key("stable upload"): - q = projectB.query("SELECT f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); - if q.getresult() != []: - reject_message = reject_message + "Rejected: can not overwrite existing copy of '%s' already in the archive.\n" % (dsc_file) - elif dsc_file[-12:] == ".orig.tar.gz": - # Check in the pool - q = projectB.query("SELECT l.path, f.filename, l.type, f.id FROM files f, location l WHERE f.filename ~ '/%s' AND l.id = f.location" % (utils.regex_safe(dsc_file))); - ql = q.getresult(); - if len(ql) > 0: - old_file = ql[0][0] + ql[0][1]; - actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); - found = old_file; - suite_type = ql[0][2]; - dsc_files[dsc_file]["files id"] = ql[0][3]; # need this for updating dsc_files in install() - # See install()... - if suite_type == "legacy" or suite_type == "legacy-mixed": - orig_tar_id = ql[0][3]; - else: - # Not there? Check in Incoming... - # [See comment above process_it() for explanation - # of why this is necessary...] - if os.access(dsc_file, os.R_OK) != 0: - files[dsc_file] = {}; - files[dsc_file]["size"] = os.stat(dsc_file)[stat.ST_SIZE]; - files[dsc_file]["md5sum"] = dsc_files[dsc_file]["md5sum"]; - files[dsc_file]["section"] = files[file]["section"]; - files[dsc_file]["priority"] = files[file]["priority"]; - files[dsc_file]["component"] = files[file]["component"]; - reprocess = 1; - return 1; - else: - reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming or in the pool.\n" % (file, dsc_file); - continue; - else: - reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file); - continue; - if actual_md5 != dsc_files[dsc_file]["md5sum"]: - reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file) - - if string.find(reject_message, "Rejected:") != -1: - return 0 - else: - return 1 + Cnf = utils.get_conf() -############################################################################### + Arguments = [('a',"automatic","Dinstall::Options::Automatic"), + ('h',"help","Dinstall::Options::Help"), + ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"), + ('n',"no-action","Dinstall::Options::No-Action"), + ('p',"no-lock", "Dinstall::Options::No-Lock"), + ('s',"no-mail", "Dinstall::Options::No-Mail"), + ('V',"version","Dinstall::Options::Version")]; -def check_md5sums (): - global reject_message; + for i in ["automatic", "help", "manual-reject", "no-action", + "no-lock", "no-mail", "version"]: + if not Cnf.has_key("Dinstall::Options::%s" % (i)): + Cnf["Dinstall::Options::%s" % (i)] = ""; - for file in files.keys(): - try: - file_handle = utils.open_file(file,"r"); - except utils.cant_open_exc: - pass; - else: - if apt_pkg.md5sum(file_handle) != files[file]["md5sum"]: - reject_message = reject_message + "Rejected: md5sum check failed for %s.\n" % (file); + changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + Options = Cnf.SubTree("Dinstall::Options") -##################################################################################################################### + Katie = katie.Katie(Cnf); + projectB = Katie.projectB; -def action (changes_filename): - byhand = confirm = suites = summary = new = ""; + changes = Katie.pkg.changes; + dsc = Katie.pkg.dsc; + dsc_files = Katie.pkg.dsc_files; + files = Katie.pkg.files; + pkg = Katie.pkg; + Subst = Katie.Subst; - # changes["distribution"] may not exist in corner cases - # (e.g. unreadable changes files) - if not changes.has_key("distribution"): - changes["distribution"] = {}; - - for suite in changes["distribution"].keys(): - if Cnf.has_key("Suite::%s::Confirm"): - confirm = confirm + suite + ", " - suites = suites + suite + ", " - confirm = confirm[:-2] - suites = suites[:-2] + return changes_files; - for file in files.keys(): - if files[file].has_key("byhand"): - byhand = 1 - summary = summary + file + " byhand\n" - elif files[file].has_key("new"): - new = 1 - summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"]) - if files[file].has_key("othercomponents"): - summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) - if files[file]["type"] == "deb": - summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file,"r")))["Description"] + '\n'; - else: - files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]) - destination = Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file - summary = summary + file + "\n to " + destination + "\n" +############################################################################### - short_summary = summary; +def usage (exit_code=0): + print """Usage: dinstall [OPTION]... [CHANGES]... + -a, --automatic automatic run + -h, --help show this help and exit. + -n, --no-action don't do anything + -p, --no-lock don't check lockfile !! for cron.daily only !! + -s, --no-mail don't send any mail + -V, --version display the version number and exit""" + sys.exit(exit_code) - # This is for direport's benefit... - f = re_fdnic.sub("\n .\n", changes.get("changes","")); +############################################################################### - if confirm or byhand or new: - summary = summary + "Changes: " + f; +def action (): + (summary, short_summary) = Katie.build_summaries(); - summary = summary + announce (short_summary, 0) - (prompt, answer) = ("", "XXX") - if Cnf["Dinstall::Options::No-Action"] or Cnf["Dinstall::Options::Automatic"]: + if Options["No-Action"] or Options["Automatic"]: answer = 'S' if string.find(reject_message, "Rejected") != -1: - if time.time()-os.path.getmtime(changes_filename) < 86400: - print "SKIP (too new)\n" + reject_message,; - prompt = "[S]kip, Manual reject, Quit ?"; - else: - print "REJECT\n" + reject_message,; - prompt = "[R]eject, Manual reject, Skip, Quit ?"; - if Cnf["Dinstall::Options::Automatic"]: - answer = 'R'; - elif new: - print "NEW to %s\n%s%s" % (suites, reject_message, summary),; - prompt = "[S]kip, New ack, Manual reject, Quit ?"; - if Cnf["Dinstall::Options::Automatic"] and Cnf["Dinstall::Options::Ack-New"]: - answer = 'N'; - elif byhand: - print "BYHAND\n" + reject_message + summary,; - prompt = "[I]nstall, Manual reject, Skip, Quit ?"; - elif confirm: - print "CONFIRM to %s\n%s%s" % (confirm, reject_message, summary), - prompt = "[I]nstall, Manual reject, Skip, Quit ?"; + print "REJECT\n" + reject_message,; + prompt = "[R]eject, Skip, Quit ?"; + if Options["Automatic"]: + answer = 'R'; else: print "INSTALL\n" + reject_message + summary,; - prompt = "[I]nstall, Manual reject, Skip, Quit ?"; - if Cnf["Dinstall::Options::Automatic"]: + prompt = "[I]nstall, Skip, Quit ?"; + if Options["Automatic"]: answer = 'I'; while string.find(prompt, answer) == -1: - print prompt,; - answer = utils.our_raw_input() - m = re_default_answer.match(prompt) + answer = utils.our_raw_input(prompt); + m = katie.re_default_answer.match(prompt); if answer == "": - answer = m.group(1) - answer = string.upper(answer[:1]) + answer = m.group(1); + answer = string.upper(answer[:1]); if answer == 'R': - reject (changes_filename, ""); - elif answer == 'M': - manual_reject (changes_filename); + do_reject (); elif answer == 'I': - install (changes_filename, summary, short_summary); - elif answer == 'N': - acknowledge_new (changes_filename, summary); + if not installing_to_stable: + install(); + else: + stable_install(summary, short_summary); elif answer == 'Q': sys.exit(0) -##################################################################################################################### +############################################################################### + +# Our reject is not really a reject, but an unaccept, but since a) the +# code for that is non-trivial (reopen bugs, unannounce etc.), b) this +# should be exteremly rare, for now we'll go with whining at our admin +# folks... + +def do_reject (): + Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; + Subst["__REJECT_MESSAGE__"] = reject_message; + Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.unaccept"); + + # Write the rejection email out as the .reason file + reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason"; + reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; + # If we fail here someone is probably trying to exploit the race + # so let's just raise an exception ... + if os.path.exists(reject_filename): + os.unlink(reject_filename); + fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + os.write(fd, reject_mail_message); + os.close(fd); + + utils.send_mail (reject_mail_message, ""); + Logger.log(["unaccepted", pkg.changes_file]); + +############################################################################### -def install (changes_filename, summary, short_summary): - global install_count, install_bytes +def install (): + global install_count, install_bytes; - # Stable uploads are a special case - if changes.has_key("stable upload"): - stable_install (changes_filename, summary, short_summary); - return; - print "Installing." - archive = utils.where_am_i(); + Logger.log(["installing changes",pkg.changes_file]); # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. projectB.query("BEGIN WORK"); @@ -638,13 +262,15 @@ def install (changes_filename, summary, short_summary): maintainer = dsc["maintainer"] maintainer = string.replace(maintainer, "'", "\\'") maintainer_id = db_access.get_or_set_maintainer_id(maintainer); + fingerprint_id = db_access.get_or_set_fingerprint_id(dsc["fingerprint"]); + install_date = time.strftime("%Y-%m-%d", time.localtime(time.time())); filename = files[file]["pool name"] + file; dsc_location_id = files[file]["location id"]; - if not files[file]["files id"]: + if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id) - projectB.query("INSERT INTO source (source, version, maintainer, file) VALUES ('%s', '%s', %d, %d)" - % (package, version, maintainer_id, files[file]["files id"])) - + projectB.query("INSERT INTO source (source, version, maintainer, file, install_date, sig_fpr) VALUES ('%s', '%s', %d, %d, '%s', %s)" + % (package, version, maintainer_id, files[file]["files id"], install_date, fingerprint_id)); + for suite in changes["distribution"].keys(): suite_id = db_access.get_suite_id(suite); projectB.query("INSERT INTO src_associations (suite, source) VALUES (%d, currval('source_id_seq'))" % (suite_id)) @@ -662,7 +288,7 @@ def install (changes_filename, summary, short_summary): if files_id == None: files_id = db_access.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id); projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)); - + # Add the .deb files to the DB for file in files.keys(): if files[file]["type"] == "deb": @@ -671,28 +297,25 @@ def install (changes_filename, summary, short_summary): maintainer = files[file]["maintainer"] maintainer = string.replace(maintainer, "'", "\\'") maintainer_id = db_access.get_or_set_maintainer_id(maintainer); + fingerprint_id = db_access.get_or_set_fingerprint_id(changes["fingerprint"]); architecture = files[file]["architecture"] architecture_id = db_access.get_architecture_id (architecture); type = files[file]["dbtype"]; - component = files[file]["component"] - source = files[file]["source"] - source_version = "" - if string.find(source, "(") != -1: - m = utils.re_extract_src_version.match(source) - source = m.group(1) - source_version = m.group(2) - if not source_version: - source_version = version + dsc_component = files[file]["component"] + source = files[file]["source package"] + source_version = files[file]["source version"]; filename = files[file]["pool name"] + file; - if not files[file]["files id"]: + if not files[file].has_key("location id") or not files[file]["location id"]: + files[file]["location id"] = db_access.get_location_id(Cnf["Dir::Pool"],files[file]["component"],utils.where_am_i()); + if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"]) source_id = db_access.get_source_id (source, source_version); if source_id: - projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, %d, '%s')" - % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type)); + projectB.query("INSERT INTO binaries (package, version, maintainer, source, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, %d, '%s', %d)" + % (package, version, maintainer_id, source_id, architecture_id, files[file]["files id"], type, fingerprint_id)); else: - projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type) VALUES ('%s', '%s', %d, %d, %d, '%s')" - % (package, version, maintainer_id, architecture_id, files[file]["files id"], type)); + projectB.query("INSERT INTO binaries (package, version, maintainer, architecture, file, type, sig_fpr) VALUES ('%s', '%s', %d, %d, %d, '%s', %d)" + % (package, version, maintainer_id, architecture_id, files[file]["files id"], type, fingerprint_id)); for suite in changes["distribution"].keys(): suite_id = db_access.get_suite_id(suite); projectB.query("INSERT INTO bin_associations (suite, bin) VALUES (%d, currval('binaries_id_seq'))" % (suite_id)); @@ -700,93 +323,133 @@ def install (changes_filename, summary, short_summary): # If the .orig.tar.gz is in a legacy directory we need to poolify # it, so that apt-get source (and anything else that goes by the # "Directory:" field in the Sources.gz file) works. - if orig_tar_id != None: + orig_tar_id = Katie.pkg.orig_tar_id; + orig_tar_location = Katie.pkg.orig_tar_location; + legacy_source_untouchable = Katie.pkg.legacy_source_untouchable; + if orig_tar_id != None and orig_tar_location == "legacy": q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)); qd = q.dictresult(); for qid in qd: + # Is this an old upload superseded by a newer -sa upload? (See check_dsc() for details) + if legacy_source_untouchable.has_key(qid["files_id"]): + continue; # First move the files to the new location legacy_filename = qid["path"]+qid["filename"]; pool_location = utils.poolify (changes["source"], files[file]["component"]); pool_filename = pool_location + os.path.basename(qid["filename"]); - destination = Cnf["Dir::PoolDir"] + pool_location + destination = Cnf["Dir::Pool"] + pool_location utils.move(legacy_filename, destination); # Then Update the DB's files table q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"])); + # If this is a sourceful diff only upload that is moving non-legacy + # cross-component we need to copy the .orig.tar.gz into the new + # component too for the same reasons as above. + # + if changes["architecture"].has_key("source") and orig_tar_id != None and \ + orig_tar_location != "legacy" and orig_tar_location != dsc_location_id: + q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id)); + ql = q.getresult()[0]; + old_filename = ql[0] + ql[1]; + file_size = ql[2]; + file_md5sum = ql[3]; + new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename); + new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id); + if new_files_id == None: + utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename); + new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id); + projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id)); + # Install the files into the pool for file in files.keys(): - if files[file].has_key("byhand"): - continue - destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file - destdir = os.path.dirname(destination) - utils.move (file, destination) - install_bytes = install_bytes + float(files[file]["size"]) + destination = Cnf["Dir::Pool"] + files[file]["pool name"] + file; + utils.move(file, destination); + Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]); + install_bytes = install_bytes + float(files[file]["size"]); # Copy the .changes file across for suite which need it. for suite in changes["distribution"].keys(): if Cnf.has_key("Suite::%s::CopyChanges" % (suite)): - utils.copy (changes_filename, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]); + utils.copy(pkg.changes_file, Cnf["Dir::Root"] + Cnf["Suite::%s::CopyChanges" % (suite)]); + # and the .katie file... + if Cnf.has_key("Suite::%s::CopyKatie" % (suite)): + utils.copy(Katie.pkg.changes_file[:-8]+".katie", Cnf["Suite::%s::CopyKatie" % (suite)]); projectB.query("COMMIT WORK"); - utils.move (changes_filename, Cnf["Dir::IncomingDir"] + 'DONE/' + os.path.basename(changes_filename)) + # Move the .changes into the 'done' directory + try: + utils.move (pkg.changes_file, os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))); + except: + utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(pkg.changes_file), sys.exc_type)); + + os.unlink(Katie.pkg.changes_file[:-8]+".katie"); + + if changes["architecture"].has_key("source") and Urgency_Logger: + Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"]); + + # Undo the work done in katie.py(accept) to help auto-building + # from accepted. + if Cnf.get("Dinstall::SpecialAcceptedAutoBuild") and \ + changes["distribution"].has_key("unstable"): + now_date = time.strftime("%Y-%m-%d %H:%M", time.localtime(time.time())); + projectB.query("BEGIN WORK"); + for file in files.keys(): + dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"], file); + # Remove it from the list of packages for later processing by apt-ftparchive + projectB.query("UPDATE unstable_accepted SET in_accepted = 'f', last_used = '%s' WHERE filename = '%s'" % (now_date, dest)); + # Update the symlink to point to the new location in the pool + pool_location = utils.poolify (changes["source"], files[file]["component"]); + src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(file)); + os.unlink(dest); + os.symlink(src, dest); + # Update last_used on any non-upload .orig.tar.gz symlink + if orig_tar_id: + # Determine the .orig.tar.gz file name + for dsc_file in dsc_files.keys(): + if dsc_file[-12:] == ".orig.tar.gz": + orig_tar_gz = os.path.join(Cnf["Dir::AcceptedAutoBuild"], dsc_file); + # Remove it from the list of packages for later processing by apt-ftparchive + projectB.query("UPDATE unstable_accepted SET in_accepted = 'f', last_used = '%s' WHERE filename = '%s'" % (now_date, orig_tar_gz)); + projectB.query("COMMIT WORK"); install_count = install_count + 1; - if not Cnf["Dinstall::Options::No-Mail"]: - mail_message = """Return-Path: %s -From: %s -To: %s -Bcc: troup@auric.debian.org -Subject: %s INSTALLED - -%s -Installing: -%s +################################################################################ -%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer) - utils.send_mail (mail_message, "") - announce (short_summary, 1) +def stable_install (summary, short_summary): + global install_count; -##################################################################################################################### - -def stable_install (changes_filename, summary, short_summary): - global install_count, install_bytes - - print "Installing to stable." - - archive = utils.where_am_i(); + print "Installing to stable."; # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. projectB.query("BEGIN WORK"); - # Add the .dsc file to the DB + # Add the source to stable (and remove it from proposed-updates) for file in files.keys(): if files[file]["type"] == "dsc": - package = dsc["source"] - version = dsc["version"] # NB: not files[file]["version"], that has no epoch + package = dsc["source"]; + version = dsc["version"]; # NB: not files[file]["version"], that has no epoch q = projectB.query("SELECT id FROM source WHERE source = '%s' AND version = '%s'" % (package, version)) - ql = q.getresult() - if ql == []: - sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s) in source table.\n" % (package, version)); - sys.exit(1); + ql = q.getresult(); + if not ql: + utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s) in source table." % (package, version)); source_id = ql[0][0]; suite_id = db_access.get_suite_id('proposed-updates'); projectB.query("DELETE FROM src_associations WHERE suite = '%s' AND source = '%s'" % (suite_id, source_id)); suite_id = db_access.get_suite_id('stable'); projectB.query("INSERT INTO src_associations (suite, source) VALUES ('%s', '%s')" % (suite_id, source_id)); - - # Add the .deb files to the DB + + # Add the binaries to stable (and remove it/them from proposed-updates) for file in files.keys(): if files[file]["type"] == "deb": - package = files[file]["package"] - version = files[file]["version"] - architecture = files[file]["architecture"] - q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)) - ql = q.getresult() - if ql == []: - sys.stderr.write("INTERNAL ERROR: couldn't find '%s' (%s for %s architecture) in binaries table.\n" % (package, version, architecture)); - sys.exit(1); + package = files[file]["package"]; + version = files[file]["version"]; + architecture = files[file]["architecture"]; + q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)); + ql = q.getresult(); + if not ql: + utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)); binary_id = ql[0][0]; suite_id = db_access.get_suite_id('proposed-updates'); projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); @@ -795,27 +458,26 @@ def stable_install (changes_filename, summary, short_summary): projectB.query("COMMIT WORK"); - utils.move (changes_filename, Cnf["Rhona::Morgue"] + os.path.basename(changes_filename)); + utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(pkg.changes_file)); - # Update the Stable ChangeLog file - - new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; - changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; + ## Update the Stable ChangeLog file + new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; + changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; if os.path.exists(new_changelog_filename): os.unlink (new_changelog_filename); - + new_changelog = utils.open_file(new_changelog_filename, 'w'); for file in files.keys(): if files[file]["type"] == "deb": new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)); - elif re_issource.match(file) != None: + elif utils.re_issource.match(file) != None: new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)); else: new_changelog.write("%s\n" % (file)); - chop_changes = re_fdnic.sub("\n", changes["changes"]); + chop_changes = katie.re_fdnic.sub("\n", changes["changes"]); new_changelog.write(chop_changes + '\n\n'); if os.access(changelog_filename, os.R_OK) != 0: - changelog = utils.open_file(changelog_filename, 'r'); + changelog = utils.open_file(changelog_filename); new_changelog.write(changelog.read()); new_changelog.close(); if os.access(changelog_filename, os.R_OK) != 0: @@ -824,340 +486,99 @@ def stable_install (changes_filename, summary, short_summary): install_count = install_count + 1; - if not Cnf["Dinstall::Options::No-Mail"]: - mail_message = """Return-Path: %s -From: %s -To: %s -Bcc: troup@auric.debian.org -Subject: %s INSTALLED into stable + if not Options["No-Mail"] and changes["architecture"].has_key("source"): + Subst["__SUITE__"] = " into stable"; + Subst["__SUMMARY__"] = summary; + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.installed"); + utils.send_mail(mail_message, ""); + Katie.announce(short_summary, 1) -%s -Installing: -%s + # Finally remove the .katie file + katie_file = os.path.join(Cnf["Suite::Proposed-Updates::CopyKatie"], os.path.basename(Katie.pkg.changes_file[:-8]+".katie")); + os.unlink(katie_file); -%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, summary, installed_footer) - utils.send_mail (mail_message, "") - announce (short_summary, 1) - -##################################################################################################################### - -def reject (changes_filename, manual_reject_mail_filename): - print "Rejecting.\n" - - base_changes_filename = os.path.basename(changes_filename); - reason_filename = re_changes.sub("reason", base_changes_filename); - reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename); - - # Move the .changes files and it's contents into REJECT/ (if we can; errors are ignored) - try: - utils.move (changes_filename, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename)); - except utils.cant_overwrite_exc: - sys.stderr.write("W: couldn't overwrite existing file '%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], base_changes_filename)); - pass; - for file in files.keys(): - if os.path.exists(file): - try: - utils.move (file, "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file)); - except utils.cant_overwrite_exc: - sys.stderr.write("W: couldn't overwrite existing file '%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], file)); - pass; - - # If this is not a manual rejection generate the .reason file and rejection mail message - if manual_reject_mail_filename == "": - if os.path.exists(reject_filename): - os.unlink(reject_filename); - fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); - os.write(fd, reject_message); - os.close(fd); - reject_mail_message = """From: %s -To: %s -Bcc: troup@auric.debian.org -Subject: %s REJECTED - -%s -=== -%s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), reject_message, reject_footer); - else: # Have a manual rejection file to use - reject_mail_message = ""; # avoid 's - - # Send the rejection mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: - utils.send_mail (reject_mail_message, manual_reject_mail_filename); - -################################################################## - -def manual_reject (changes_filename): - # Build up the rejection email - user_email_address = string.replace(string.split(pwd.getpwuid(os.getuid())[4],',')[0], '.', '') - user_email_address = user_email_address + " <%s@%s>" % (pwd.getpwuid(os.getuid())[0], Cnf["Dinstall::MyHost"]) - manual_reject_message = Cnf.get("Dinstall::Options::Manual-Reject", "") - - reject_mail_message = """From: %s -Cc: %s -To: %s -Bcc: troup@auric.debian.org -Subject: %s REJECTED - -%s -%s -=== -%s""" % (user_email_address, Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], os.path.basename(changes_filename), manual_reject_message, reject_message, reject_footer) - - # Write the rejection email out as the .reason file - reason_filename = re_changes.sub("reason", os.path.basename(changes_filename)); - reject_filename = "%s/REJECT/%s" % (Cnf["Dir::IncomingDir"], reason_filename) - if os.path.exists(reject_filename): - os.unlink(reject_filename); - fd = os.open(reject_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); - os.write(fd, reject_mail_message); - os.close(fd); - - # If we weren't given one, spawn an editor so the user can add one in - if manual_reject_message == "": - result = os.system("vi +6 %s" % (reject_file)) - if result != 0: - sys.stderr.write ("vi invocation failed for `%s'!" % (reject_file)) - sys.exit(result) - - # Then process it as if it were an automatic rejection - reject (changes_filename, reject_filename) - -##################################################################################################################### - -def acknowledge_new (changes_filename, summary): - global new_ack_new; - - changes_filename = os.path.basename(changes_filename); - - new_ack_new[changes_filename] = 1; - - if new_ack_old.has_key(changes_filename): - print "Ack already sent."; - return; - - print "Sending new ack."; - if not Cnf["Dinstall::Options::No-Mail"]: - new_ack_message = """Return-Path: %s -From: %s -To: %s -Bcc: troup@auric.debian.org -Subject: %s is NEW - -%s -%s""" % (Cnf["Dinstall::MyEmailAddress"], Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes_filename, summary, new_ack_footer); - utils.send_mail(new_ack_message,""); - -##################################################################################################################### - -def announce (short_summary, action): - # Only do announcements for source uploads with a recent dpkg-dev installed - if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"): - return "" - - lists_done = {} - summary = "" - - for dist in changes["distribution"].keys(): - list = Cnf.Find("Suite::%s::Announce" % (dist)) - if list == None or lists_done.has_key(list): - continue - lists_done[list] = 1 - summary = summary + "Announcing to %s\n" % (list) - - if action: - mail_message = """Return-Path: %s -From: %s -To: %s -Bcc: troup@auric.debian.org -Subject: Installed %s %s (%s) - -%s - -Installed: -%s -""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], list, changes["source"], changes["version"], string.join(changes["architecture"].keys(), ' ' ), - changes["filecontents"], short_summary) - utils.send_mail (mail_message, "") - - (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"])); - bugs = changes["closes"].keys() - bugs.sort() - if dsc_name == changes["maintainername"]: - summary = summary + "Closing bugs: " - for bug in bugs: - summary = summary + "%s " % (bug) - if action: - mail_message = """Return-Path: %s -From: %s -To: %s-close@bugs.debian.org -Bcc: troup@auric.debian.org -Subject: Bug#%s: fixed in %s %s - -We believe that the bug you reported is fixed in the latest version of -%s, which has been installed in the Debian FTP archive: - -%s""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], bug, bug, changes["source"], changes["version"], changes["source"], short_summary) - - if changes["distribution"].has_key("stable"): - mail_message = mail_message + """Note that this package is not part of the released stable Debian -distribution. It may have dependencies on other unreleased software, -or other instabilities. Please take care if you wish to install it. -The update will eventually make its way into the next released Debian -distribution.""" - - mail_message = mail_message + """A summary of the changes between this version and the previous one is -attached. - -Thank you for reporting the bug, which will now be closed. If you -have further comments please address them to %s@bugs.debian.org, -and the maintainer will reopen the bug report if appropriate. - -Debian distribution maintenance software -pp. -%s (supplier of updated %s package) - -(This message was generated automatically at their request; if you -believe that there is a problem with it please contact the archive -administrators by mailing ftpmaster@debian.org) - - -%s""" % (bug, changes["maintainer"], changes["source"], changes["filecontents"]) - - utils.send_mail (mail_message, "") - else: # NMU - summary = summary + "Setting bugs to severity fixed: " - control_message = "" - for bug in bugs: - summary = summary + "%s " % (bug) - control_message = control_message + "severity %s fixed\n" % (bug) - if action and control_message != "": - mail_message = """Return-Path: %s -From: %s -To: control@bugs.debian.org -Bcc: troup@auric.debian.org, %s -Subject: Fixed in NMU of %s %s - -%s -quit - -This message was generated automatically in response to a -non-maintainer upload. The .changes file follows. - -%s -""" % (Cnf["Dinstall::MyEmailAddress"], changes["maintainer822"], changes["maintainer822"], changes["source"], changes["version"], control_message, changes["filecontents"]) - utils.send_mail (mail_message, "") - summary = summary + "\n" - - return summary - -############################################################################### - -# reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in -# Incoming. -1 will reference the .orig.tar.gz, but -2 will not. -# dsccheckdistrib() can find the .orig.tar.gz but it will not have -# processed it during it's checks of -2. If -1 has been deleted or -# otherwise not checked by da-install, the .orig.tar.gz will not have -# been checked at all. To get round this, we force the .orig.tar.gz -# into the .changes structure and reprocess the .changes file. +################################################################################ def process_it (changes_file): - global reprocess, orig_tar_id, changes, dsc, dsc_files, files; + global reject_message; - reprocess = 1; - orig_tar_id = None; - # Reset some globals - changes = {}; - dsc = {}; - dsc_files = {}; - files = {}; - orig_tar_id = None; + reject_message = ""; # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. - changes_file = os.path.abspath(changes_file); + pkg.changes_file = os.path.abspath(changes_file); # And since handling of installs to stable munges with the CWD; # save and restore it. - cwd = os.getcwd(); - - check_signature (changes_file); - check_changes (changes_file); - while reprocess: - reprocess = 0; - check_files (); - check_md5sums (); - check_dsc (); - - action(changes_file); + pkg.directory = os.getcwd(); + + if installing_to_stable: + old = Katie.pkg.changes_file; + Katie.pkg.changes_file = os.path.basename(old); + os.chdir(Cnf["Suite::Proposed-Updates::CopyKatie"]); + + Katie.init_vars(); + Katie.update_vars(); + Katie.update_subst(); + + if installing_to_stable: + Katie.pkg.changes_file = old; + + check(); + action(); # Restore CWD - os.chdir(cwd); + os.chdir(pkg.directory); ############################################################################### def main(): - global Cnf, projectB, reject_message, install_bytes, new_ack_old + global projectB, Logger, Urgency_Logger, installing_to_stable; - apt_pkg.init(); - - Cnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + changes_files = init(); - Arguments = [('a',"automatic","Dinstall::Options::Automatic"), - ('d',"debug","Dinstall::Options::Debug", "IntVal"), - ('h',"help","Dinstall::Options::Help"), - ('k',"ack-new","Dinstall::Options::Ack-New"), - ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"), - ('n',"no-action","Dinstall::Options::No-Action"), - ('p',"no-lock", "Dinstall::Options::No-Lock"), - ('r',"no-version-check", "Dinstall::Options::No-Version-Check"), - ('s',"no-mail", "Dinstall::Options::No-Mail"), - ('u',"override-distribution", "Dinstall::Options::Override-Distribution", "HasArg"), - ('v',"version","Dinstall::Options::Version")]; - - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - - if Cnf["Dinstall::Options::Help"]: - usage(0); - - if Cnf["Dinstall::Options::Version"]: - print "katie version 0.0000000000"; - usage(0); + if Options["Help"]: + usage(); - postgresql_user = None; # Default == Connect as user running program. + if Options["Version"]: + print "katie %s" % (katie_version); + sys.exit(0); # -n/--dry-run invalidates some other options which would involve things happening - if Cnf["Dinstall::Options::No-Action"]: - Cnf["Dinstall::Options::Automatic"] = "" - Cnf["Dinstall::Options::Ack-New"] = "" - postgresql_user = Cnf["DB::ROUser"]; - - projectB = pg.connect('projectb', Cnf["DB::Host"], int(Cnf["DB::Port"]), None, None, postgresql_user); - - db_access.init(Cnf, projectB); + if Options["No-Action"]: + Options["Automatic"] = ""; # Check that we aren't going to clash with the daily cron job - if os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Cnf["Dinstall::Options::No-Lock"]: - sys.stderr.write("Archive maintenance in progress. Try again later.\n"); - sys.exit(2); - - # Obtain lock if not in no-action mode + if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: + utils.fubar("Archive maintenance in progress. Try again later."); - if not Cnf["Dinstall::Options::No-Action"]: - lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR); + # If running from within proposed-updates; assume an install to stable + if string.find(os.getcwd(), 'proposed-updates') != -1: + installing_to_stable = 1; + + # Obtain lock if not in no-action mode and initialize the log + if not Options["No-Action"]: + lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT); fcntl.lockf(lock_fd, FCNTL.F_TLOCK); + Logger = Katie.Logger = logging.Logger(Cnf, "katie"); + if not installing_to_stable and Cnf.get("Dir::UrgencyLog"): + Urgency_Logger = Urgency_Log(Cnf); + + # Initialize the substitution template mapping global + bcc = "X-Katie: %s" % (katie_version); + if Cnf.has_key("Dinstall::Bcc"): + Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); + else: + Subst["__BCC__"] = bcc; + if Cnf.has_key("Dinstall::StableRejector"): + Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"]; - # Read in the list of already-acknowledged NEW packages - new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'r'); - new_ack_old = {}; - for line in new_ack_list.readlines(): - new_ack_old[line[:-1]] = 1; - new_ack_list.close(); + # Sort the .changes files so that we process sourceful ones first + changes_files.sort(utils.changes_compare); # Process the changes files for changes_file in changes_files: - reject_message = "" print "\n" + changes_file; process_it (changes_file); @@ -1166,15 +587,14 @@ def main(): if install_count > 1: sets = "sets" sys.stderr.write("Installed %d package %s, %s.\n" % (install_count, sets, utils.size_type(int(install_bytes)))); + Logger.log(["total",install_count,install_bytes]); - # Write out the list of already-acknowledged NEW packages - if Cnf["Dinstall::Options::Ack-New"]: - new_ack_list = utils.open_file(Cnf["Dinstall::NewAckList"],'w') - for i in new_ack_new.keys(): - new_ack_list.write(i+'\n') - new_ack_list.close() - - -if __name__ == '__main__': - main() + if not Options["No-Action"]: + Logger.close(); + if Urgency_Logger: + Urgency_Logger.close(); + +############################################################################### +if __name__ == '__main__': + main();