X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=kelly;h=d28d1ace28755bbecd69f90a667b2954fc375b80;hb=9540d873fa78598454af57f5f8a4875969ed0439;hp=80f1b204c272f9a65cd86642c5ddf4bfe717e505;hpb=c846e77a848d60dd115f00faa0d9a854161d99eb;p=dak.git diff --git a/kelly b/kelly index 80f1b204..d28d1ace 100755 --- a/kelly +++ b/kelly @@ -1,8 +1,8 @@ #!/usr/bin/env python -# Installs Debian packages -# Copyright (C) 2000, 2001, 2002 James Troup -# $Id: kelly,v 1.1 2002-10-16 02:47:32 troup Exp $ +# Installs Debian packages from queue/accepted into the pool +# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup +# $Id: kelly,v 1.18 2005-12-17 10:57:03 rmurray Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,8 +18,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# Originally based on dinstall by Guy Maor - ############################################################################### # Cartman: "I'm trying to make the best of a bad situation, I don't @@ -32,14 +30,14 @@ ############################################################################### -import FCNTL, fcntl, os, sys, time; +import errno, fcntl, os, sys, time, re; import apt_pkg; import db_access, katie, logging, utils; ############################################################################### # Globals -kelly_version = "$Revision: 1.1 $"; +kelly_version = "$Revision: 1.18 $"; Cnf = None; Options = None; @@ -109,24 +107,44 @@ def reject (str, prefix="Rejected: "): # frozen between accept and our run time. def check(): + propogate={} + nopropogate={} for file in files.keys(): + # The .orig.tar.gz can disappear out from under us is it's a + # duplicate of one in the archive. + if not files.has_key(file): + continue; # Check that the source still exists if files[file]["type"] == "deb": source_version = files[file]["source version"]; source_package = files[file]["source package"]; if not changes["architecture"].has_key("source") \ - and not Katie.source_exists(source_package, source_version): + and not Katie.source_exists(source_package, source_version, changes["distribution"].keys()): reject("no source found for %s %s (%s)." % (source_package, source_version, file)); # Version and file overwrite checks if not installing_to_stable: if files[file]["type"] == "deb": - reject(Katie.check_binary_against_db(file)); + reject(Katie.check_binary_against_db(file), ""); elif files[file]["type"] == "dsc": - reject(Katie.check_source_against_db(file)); + reject(Katie.check_source_against_db(file), ""); (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); - reject(reject_msg); + reject(reject_msg, ""); + + # propogate in the case it is in the override tables: + if changes.has_key("propdistribution"): + for suite in changes["propdistribution"].keys(): + if Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + propogate[suite] = 1 + else: + nopropogate[suite] = 1 + for suite in propogate.keys(): + if suite in nopropogate: + continue + changes["distribution"][suite] = 1 + + for file in files.keys(): # Check the package is still in the override tables for suite in changes["distribution"].keys(): if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): @@ -141,20 +159,25 @@ def init(): Arguments = [('a',"automatic","Dinstall::Options::Automatic"), ('h',"help","Dinstall::Options::Help"), - ('m',"manual-reject","Dinstall::Options::Manual-Reject", "HasArg"), ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), ('s',"no-mail", "Dinstall::Options::No-Mail"), ('V',"version","Dinstall::Options::Version")]; - for i in ["automatic", "help", "manual-reject", "no-action", - "no-lock", "no-mail", "version"]: + for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version"]: if not Cnf.has_key("Dinstall::Options::%s" % (i)): Cnf["Dinstall::Options::%s" % (i)] = ""; changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); Options = Cnf.SubTree("Dinstall::Options") + if Options["Help"]: + usage(); + + if Options["Version"]: + print "kelly %s" % (kelly_version); + sys.exit(0); + Katie = katie.Katie(Cnf); projectB = Katie.projectB; @@ -170,7 +193,7 @@ def init(): ############################################################################### def usage (exit_code=0): - print """Usage: dinstall [OPTION]... [CHANGES]... + print """Usage: kelly [OPTION]... [CHANGES]... -a, --automatic automatic run -h, --help show this help and exit. -n, --no-action don't do anything @@ -194,7 +217,8 @@ def action (): if Options["Automatic"]: answer = 'R'; else: - print "INSTALL\n" + reject_message + summary,; + print "INSTALL to " + ", ".join(changes["distribution"].keys()) + print reject_message + summary,; prompt = "[I]nstall, Skip, Quit ?"; if Options["Automatic"]: answer = 'I'; @@ -240,7 +264,7 @@ def do_reject (): os.write(fd, reject_mail_message); os.close(fd); - utils.send_mail (reject_mail_message, ""); + utils.send_mail(reject_mail_message); Logger.log(["unaccepted", pkg.changes_file]); ############################################################################### @@ -266,6 +290,7 @@ def install (): fingerprint_id = db_access.get_or_set_fingerprint_id(dsc["fingerprint"]); install_date = time.strftime("%Y-%m-%d"); filename = files[file]["pool name"] + file; + dsc_component = files[file]["component"]; dsc_location_id = files[file]["location id"]; if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], dsc_location_id) @@ -302,7 +327,6 @@ def install (): architecture = files[file]["architecture"] architecture_id = db_access.get_architecture_id (architecture); type = files[file]["dbtype"]; - dsc_component = files[file]["component"] source = files[file]["source package"] source_version = files[file]["source version"]; filename = files[file]["pool name"] + file; @@ -327,7 +351,7 @@ def install (): orig_tar_id = Katie.pkg.orig_tar_id; orig_tar_location = Katie.pkg.orig_tar_location; legacy_source_untouchable = Katie.pkg.legacy_source_untouchable; - if orig_tar_id != None and orig_tar_location == "legacy": + if orig_tar_id and orig_tar_location == "legacy": q = projectB.query("SELECT DISTINCT ON (f.id) l.path, f.filename, f.id as files_id, df.source, df.id as dsc_files_id, f.size, f.md5sum FROM files f, dsc_files df, location l WHERE df.source IN (SELECT source FROM dsc_files WHERE file = %s) AND f.id = df.file AND l.id = f.location AND (l.type = 'legacy' OR l.type = 'legacy-mixed')" % (orig_tar_id)); qd = q.dictresult(); for qid in qd: @@ -347,7 +371,7 @@ def install (): # cross-component we need to copy the .orig.tar.gz into the new # component too for the same reasons as above. # - if changes["architecture"].has_key("source") and orig_tar_id != None and \ + if changes["architecture"].has_key("source") and orig_tar_id and \ orig_tar_location != "legacy" and orig_tar_location != dsc_location_id: q = projectB.query("SELECT l.path, f.filename, f.size, f.md5sum FROM files f, location l WHERE f.id = %s AND f.location = l.id" % (orig_tar_id)); ql = q.getresult()[0]; @@ -369,26 +393,26 @@ def install (): install_bytes += float(files[file]["size"]); # Copy the .changes file across for suite which need it. - copy_changes_p = copy_katie_p = 0; + copy_changes = {}; + copy_katie = {}; for suite in changes["distribution"].keys(): if Cnf.has_key("Suite::%s::CopyChanges" % (suite)): - copy_changes_p = 1; + copy_changes[Cnf["Suite::%s::CopyChanges" % (suite)]] = ""; # and the .katie file... if Cnf.has_key("Suite::%s::CopyKatie" % (suite)): - copy_katie_p = 1; - if copy_changes_p: - utils.copy(pkg.changes_file, Cnf["Dir::Root"] + Cnf["Suite::%s::CopyChanges" % (suite)]); - if copy_katie_p: - utils.copy(Katie.pkg.changes_file[:-8]+".katie", Cnf["Suite::%s::CopyKatie" % (suite)]); + copy_katie[Cnf["Suite::%s::CopyKatie" % (suite)]] = ""; + for dest in copy_changes.keys(): + utils.copy(pkg.changes_file, Cnf["Dir::Root"] + dest); + for dest in copy_katie.keys(): + utils.copy(Katie.pkg.changes_file[:-8]+".katie", dest); projectB.query("COMMIT WORK"); # Move the .changes into the 'done' directory - try: - utils.move (pkg.changes_file, os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))); - except: - utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(pkg.changes_file), sys.exc_type)); + utils.move (pkg.changes_file, + os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))); + # Remove the .katie file os.unlink(Katie.pkg.changes_file[:-8]+".katie"); if changes["architecture"].has_key("source") and Urgency_Logger: @@ -398,18 +422,18 @@ def install (): # from accepted. projectB.query("BEGIN WORK"); for suite in changes["distribution"].keys(): - if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"): + if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): continue; now_date = time.strftime("%Y-%m-%d %H:%M"); suite_id = db_access.get_suite_id(suite); - dest_dir = Cnf["Dir::AcceptedAutoBuild"]; - if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + dest_dir = Cnf["Dir::QueueBuild"]; + if Cnf.FindB("Dinstall::SecurityQueueBuild"): dest_dir = os.path.join(dest_dir, suite); for file in files.keys(): dest = os.path.join(dest_dir, file); # Remove it from the list of packages for later processing by apt-ftparchive - projectB.query("UPDATE accepted_autobuild SET in_accepted = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id)); - if not Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, dest, suite_id)); + if not Cnf.FindB("Dinstall::SecurityQueueBuild"): # Update the symlink to point to the new location in the pool pool_location = utils.poolify (changes["source"], files[file]["component"]); src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(file)); @@ -423,7 +447,7 @@ def install (): if dsc_file.endswith(".orig.tar.gz"): orig_tar_gz = os.path.join(dest_dir, dsc_file); # Remove it from the list of packages for later processing by apt-ftparchive - projectB.query("UPDATE accepted_autobuild SET in_accepted = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id)); + projectB.query("UPDATE queue_build SET in_queue = 'f', last_used = '%s' WHERE filename = '%s' AND suite = %s" % (now_date, orig_tar_gz, suite_id)); projectB.query("COMMIT WORK"); # Finally... @@ -436,7 +460,8 @@ def stable_install (summary, short_summary): print "Installing to stable."; - # Begin a transaction; if we bomb out anywhere between here and the COMMIT WORK below, the DB will not be changed. + # Begin a transaction; if we bomb out anywhere between here and + # the COMMIT WORK below, the DB won't be changed. projectB.query("BEGIN WORK"); # Add the source to stable (and remove it from proposed-updates) @@ -457,18 +482,34 @@ def stable_install (summary, short_summary): # Add the binaries to stable (and remove it/them from proposed-updates) for file in files.keys(): if files[file]["type"] == "deb": + binNMU = 0 package = files[file]["package"]; version = files[file]["version"]; architecture = files[file]["architecture"]; q = projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND b.architecture = a.id" % (package, version, architecture)); ql = q.getresult(); if not ql: - utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)); - binary_id = ql[0][0]; - suite_id = db_access.get_suite_id('proposed-updates'); - projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); - suite_id = db_access.get_suite_id('stable'); - projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)); + suite_id = db_access.get_suite_id('proposed-updates'); + que = "SELECT b.version FROM binaries b JOIN bin_associations ba ON (b.id = ba.bin) JOIN suite su ON (ba.suite = su.id) WHERE b.package = '%s' AND (ba.suite = '%s')" % (package, suite_id); + q = projectB.query(que) + + # Reduce the query results to a list of version numbers + ql = map(lambda x: x[0], q.getresult()); + if not ql: + utils.fubar("[INTERNAL ERROR] couldn't find '%s' (%s for %s architecture) in binaries table." % (package, version, architecture)); + else: + for x in ql: + if re.match(re.compile(r"%s((\.0)?\.)|(\+b)\d+$" % re.escape(version)),x): + binNMU = 1 + break + if not binNMU: + binary_id = ql[0][0]; + suite_id = db_access.get_suite_id('proposed-updates'); + projectB.query("DELETE FROM bin_associations WHERE suite = '%s' AND bin = '%s'" % (suite_id, binary_id)); + suite_id = db_access.get_suite_id('stable'); + projectB.query("INSERT INTO bin_associations (suite, bin) VALUES ('%s', '%s')" % (suite_id, binary_id)); + else: + del files[file] projectB.query("COMMIT WORK"); @@ -484,7 +525,7 @@ def stable_install (summary, short_summary): for file in files.keys(): if files[file]["type"] == "deb": new_changelog.write("stable/%s/binary-%s/%s\n" % (files[file]["component"], files[file]["architecture"], file)); - elif utils.re_issource.match(file) != None: + elif utils.re_issource.match(file): new_changelog.write("stable/%s/source/%s\n" % (files[file]["component"], file)); else: new_changelog.write("%s\n" % (file)); @@ -504,7 +545,7 @@ def stable_install (summary, short_summary): Subst["__SUITE__"] = " into stable"; Subst["__SUMMARY__"] = summary; mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/kelly.installed"); - utils.send_mail(mail_message, ""); + utils.send_mail(mail_message); Katie.announce(short_summary, 1) # Finally remove the .katie file @@ -551,13 +592,6 @@ def main(): changes_files = init(); - if Options["Help"]: - usage(); - - if Options["Version"]: - print "kelly %s" % (kelly_version); - sys.exit(0); - # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = ""; @@ -574,8 +608,14 @@ def main(): # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT); - fcntl.lockf(lock_fd, FCNTL.F_TLOCK); - Logger = Katie.Logger = logging.Logger(Cnf, "katie"); + try: + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB); + except IOError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': + utils.fubar("Couldn't obtain lock; assuming another kelly is already running."); + else: + raise; + Logger = Katie.Logger = logging.Logger(Cnf, "kelly"); if not installing_to_stable and Cnf.get("Dir::UrgencyLog"): Urgency_Logger = Urgency_Log(Cnf); @@ -585,8 +625,6 @@ def main(): Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); else: Subst["__BCC__"] = bcc; - if Cnf.has_key("Dinstall::StableRejector"): - Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"]; # Sort the .changes files so that we process sourceful ones first changes_files.sort(utils.changes_compare);