X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=katie.py;h=7f428318ffea66b24e305932eda4975e1730a04a;hb=9540d873fa78598454af57f5f8a4875969ed0439;hp=637de2779e236687f0742b19150eb91d0a28e152;hpb=0f6538cf36def865bd8fc16aeef499baf5a5030a;p=dak.git diff --git a/katie.py b/katie.py index 637de277..7f428318 100644 --- a/katie.py +++ b/katie.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # Utility functions for katie -# Copyright (C) 2001, 2002, 2003, 2004 James Troup -# $Id: katie.py,v 1.49 2004-11-27 13:29:17 troup Exp $ +# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: katie.py,v 1.59 2005-12-17 10:57:03 rmurray Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,10 +30,8 @@ from types import *; re_isanum = re.compile (r"^\d+$"); re_default_answer = re.compile(r"\[(.*)\]"); -re_fdnic = re.compile("\n\n"); -re_bin_only_nmu_of_mu = re.compile("\.\d+\.\d+$"); -re_bin_only_nmu_of_nmu = re.compile("\.\d+$"); - +re_fdnic = re.compile(r"\n\n"); +re_bin_only_nmu = re.compile(r"\+b\d+$"); ############################################################################### # Convenience wrapper to carry around all the package information in @@ -176,7 +174,7 @@ class Katie: "closes", "changes" ]: d_changes[i] = changes[i]; # Optional changes fields - for i in [ "changed-by", "filecontents", "format", "lisa note" ]: + for i in [ "changed-by", "filecontents", "format", "lisa note", "distribution-version" ]: if changes.has_key(i): d_changes[i] = changes[i]; ## dsc @@ -265,7 +263,9 @@ class Katie: if files[file].has_key("othercomponents"): summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) if files[file]["type"] == "deb": - summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n'; + deb_fh = utils.open_file(file) + summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n'; + deb_fh.close() else: files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"]) destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file @@ -449,26 +449,41 @@ distribution."""; changes_file[:-8]+".debinfo"); os.rename(temp_filename, filename); - ## Special support to enable clean auto-building of accepted packages + self.queue_build("accepted", Cnf["Dir::Queue::Accepted"]) + + ########################################################################### + + def queue_build (self, queue, path): + Cnf = self.Cnf + Subst = self.Subst + files = self.pkg.files + changes = self.pkg.changes + changes_file = self.pkg.changes_file + dsc = self.pkg.dsc + file_keys = files.keys() + + ## Special support to enable clean auto-building of queued packages + queue_id = db_access.get_or_set_queue_id(queue) + self.projectB.query("BEGIN WORK"); for suite in changes["distribution"].keys(): - if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"): + if suite not in Cnf.ValueList("Dinstall::QueueBuildSuites"): continue; suite_id = db_access.get_suite_id(suite); - dest_dir = Cnf["Dir::AcceptedAutoBuild"]; - if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + dest_dir = Cnf["Dir::QueueBuild"]; + if Cnf.FindB("Dinstall::SecurityQueueBuild"): dest_dir = os.path.join(dest_dir, suite); for file in file_keys: - src = os.path.join(Cnf["Dir::Queue::Accepted"], file); + src = os.path.join(path, file); dest = os.path.join(dest_dir, file); - if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + if Cnf.FindB("Dinstall::SecurityQueueBuild"): # Copy it since the original won't be readable by www-data utils.copy(src, dest); else: # Create a symlink to it os.symlink(src, dest); # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest)); + self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)); # If the .orig.tar.gz is in the pool, create a symlink to # it (if one doesn't already exist) if self.pkg.orig_tar_id: @@ -487,10 +502,10 @@ distribution."""; src = os.path.join(ql[0][0], ql[0][1]); os.symlink(src, dest); # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest)); + self.projectB.query("INSERT INTO queue_build (suite, queue, filename, in_queue) VALUES (%s, %s, '%s', 't')" % (suite_id, queue_id, dest)); # if it does, update things to ensure it's not removed prematurely else: - self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id)); + self.projectB.query("UPDATE queue_build SET in_queue = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id)); self.projectB.query("COMMIT WORK"); @@ -551,7 +566,7 @@ distribution."""; continue; dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file); try: - os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); except OSError, e: # File exists? Let's try and move it to the morgue if errno.errorcode[e.errno] == 'EEXIST': @@ -565,7 +580,7 @@ distribution."""; return; utils.move(dest_file, morgue_file, perms=0660); try: - os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); except OSError, e: # Likewise utils.warn("**WARNING** failed to claim %s in the reject directory." % (file)); @@ -575,6 +590,7 @@ distribution."""; # If we got here, we own the destination file, so we can # safely overwrite it. utils.move(file, dest_file, 1, perms=0660); + os.close(dest_fd) ########################################################################### @@ -587,9 +603,9 @@ distribution."""; answer = 'E'; while answer == 'E': os.system("%s %s" % (editor, temp_filename)) - file = utils.open_file(temp_filename); - reject_message = "".join(file.readlines()); - file.close(); + temp_fh = utils.open_file(temp_filename); + reject_message = "".join(temp_fh.readlines()); + temp_fh.close(); print "Reject message:"; print utils.prefix_multi_line_string(reject_message," ",include_blank_lines=1); prompt = "[R]eject, Edit, Abandon, Quit ?" @@ -623,13 +639,13 @@ distribution."""; # so let's just raise an exception ... if os.path.exists(reason_filename): os.unlink(reason_filename); - reason_file = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); + reason_fd = os.open(reason_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); if not manual: Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; Subst["__MANUAL_REJECT_MESSAGE__"] = ""; Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)"; - os.write(reason_file, reject_message); + os.write(reason_fd, reject_message); reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); else: # Build up the rejection email @@ -640,9 +656,9 @@ distribution."""; Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); # Write the rejection email out as the .reason file - os.write(reason_file, reject_mail_message); + os.write(reason_fd, reject_mail_message); - os.close(reason_file); + os.close(reason_fd) # Send the rejection mail if appropriate if not Cnf["Dinstall::Options::No-Mail"]: @@ -657,8 +673,7 @@ distribution."""; # upload being processed. # # (1) exact match => 1.0-3 - # (2) Bin-only NMU of an MU => 1.0-3.0.1 - # (3) Bin-only NMU of a sourceful-NMU => 1.0-3.1.1 + # (2) Bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 def source_exists (self, package, source_version, suites = ["any"]): okay = 1 @@ -691,17 +706,13 @@ distribution."""; continue # Try (2) - orig_source_version = re_bin_only_nmu_of_mu.sub('', source_version) - if orig_source_version in ql: - continue - - # Try (3) - orig_source_version = re_bin_only_nmu_of_nmu.sub('', source_version) + orig_source_version = re_bin_only_nmu.sub('', source_version) if orig_source_version in ql: continue # No source found... okay = 0 + break return okay ################################################################################ @@ -726,7 +737,7 @@ distribution."""; type_id = db_access.get_override_type_id(type); # FIXME: nasty non-US speficic hack - if component.lower().startswith() == "non-us/": + if component.lower().startswith("non-us/"): component = component[7:]; q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" @@ -759,6 +770,17 @@ distribution."""; ################################################################################ + def get_anyversion(self, query_result, suite): + anyversion=None + anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite)) + for (v, s) in query_result: + if s in [ string.lower(x) for x in anysuite ]: + if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0: + anyversion=v + return anyversion + + ################################################################################ + def cross_suite_version_check(self, query_result, file, new_version): """Ensure versions are newer than existing packages in target suites and that cross-suite version checking rules as @@ -779,7 +801,48 @@ distribution."""; self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); if suite in must_be_older_than and \ apt_pkg.VersionCompare(new_version, existent_version) > -1: - self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); + ch = self.pkg.changes + cansave = 0 + if ch.get('distribution-version', {}).has_key(suite): + # we really use the other suite, ignoring the conflicting one ... + addsuite = ch["distribution-version"][suite] + + add_version = self.get_anyversion(query_result, addsuite) + target_version = self.get_anyversion(query_result, target_suite) + + if not add_version: + # not add_version can only happen if we map to a suite + # that doesn't enhance the suite we're propup'ing from. + # so "propup-ver x a b c; map a d" is a problem only if + # d doesn't enhance a. + # + # i think we could always propagate in this case, rather + # than complaining. either way, this isn't a REJECT issue + # + # And - we really should complain to the dorks who configured dak + self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ") + self.pkg.changes.setdefault("propdistribution", {}) + self.pkg.changes["propdistribution"][addsuite] = 1 + cansave = 1 + elif not target_version: + # not targets_version is true when the package is NEW + # we could just stick with the "...old version..." REJECT + # for this, I think. + self.reject("Won't propogate NEW packages.") + elif apt_pkg.VersionCompare(new_version, add_version) < 0: + # propogation would be redundant. no need to reject though. + self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ") + cansave = 1 + elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \ + apt_pkg.VersionCompare(add_version, target_version) >= 0: + # propogate!! + self.reject("Propogating upload to %s" % (addsuite), "Warning: ") + self.pkg.changes.setdefault("propdistribution", {}) + self.pkg.changes["propdistribution"][addsuite] = 1 + cansave = 1 + + if not cansave: + self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)) ################################################################################ @@ -901,7 +964,9 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su if len(ql) > 1: for i in ql: old_file = i[0] + i[1]; - actual_md5 = apt_pkg.md5sum(utils.open_file(old_file)); + old_file_fh = utils.open_file(old_file) + actual_md5 = apt_pkg.md5sum(old_file_fh); + old_file_fh.close() actual_size = os.stat(old_file)[stat.ST_SIZE]; if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]): x = i; @@ -909,7 +974,9 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su legacy_source_untouchable[i[3]] = ""; old_file = x[0] + x[1]; - actual_md5 = apt_pkg.md5sum(utils.open_file(old_file)); + old_file_fh = utils.open_file(old_file) + actual_md5 = apt_pkg.md5sum(old_file_fh); + old_file_fh.close() actual_size = os.stat(old_file)[stat.ST_SIZE]; found = old_file; suite_type = x[2]; @@ -932,7 +999,9 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su for dir in [ "Accepted", "New", "Byhand" ]: in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file); if os.path.exists(in_otherdir): - actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir)); + in_otherdir_fh = utils.open_file(in_otherdir) + actual_md5 = apt_pkg.md5sum(in_otherdir_fh); + in_otherdir_fh.close() actual_size = os.stat(in_otherdir)[stat.ST_SIZE]; found = in_otherdir; self.pkg.orig_tar_gz = in_otherdir;