X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=katie.py;h=c195e52ac0713734ba68c8210028ceed515a7e28;hb=786a2024fd2fcf90ba8d64001ca7d13ed43b7cba;hp=0031d5645c03b719f2b695dd2ad1a928b9077074;hpb=23c6325551ac3c23cf87fbcda6d2afac01876b36;p=dak.git diff --git a/katie.py b/katie.py index 0031d564..c195e52a 100644 --- a/katie.py +++ b/katie.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # Utility functions for katie -# Copyright (C) 2001, 2002 James Troup -# $Id: katie.py,v 1.19 2002-05-10 00:24:33 troup Exp $ +# Copyright (C) 2001, 2002, 2003 James Troup +# $Id: katie.py,v 1.29 2003-02-07 14:51:48 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +25,6 @@ import utils, db_access; import apt_inst, apt_pkg; from types import *; -from string import lower; ############################################################################### @@ -57,7 +56,7 @@ class nmu_p: filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"]; file = utils.open_file(filename); for line in file.readlines(): - line = lower(string.strip(utils.re_comments.sub('', line))); + line = utils.re_comments.sub('', line).lower().strip(); if line != "": self.group_maint[line] = 1; file.close(); @@ -67,23 +66,23 @@ class nmu_p: changes = pkg.changes; dsc = pkg.dsc; - (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (lower(dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]))); + (dsc_rfc822, dsc_name, dsc_email) = utils.fix_maintainer (dsc.get("maintainer",Cnf["Dinstall::MyEmailAddress"]).lower()); # changes["changedbyname"] == dsc_name is probably never true, but better safe than sorry - if dsc_name == lower(changes["maintainername"]) and \ - (changes["changedby822"] == "" or lower(changes["changedbyname"]) == dsc_name): + if dsc_name == changes["maintainername"].lower() and \ + (changes["changedby822"] == "" or changes["changedbyname"].lower() == dsc_name): return 0; if dsc.has_key("uploaders"): - uploaders = string.split(lower(dsc["uploaders"]), ","); + uploaders = dsc["uploaders"].lower().split(","); uploadernames = {}; for i in uploaders: - (rfc822, name, email) = utils.fix_maintainer (string.strip(i)); + (rfc822, name, email) = utils.fix_maintainer (i.strip()); uploadernames[name] = ""; - if uploadernames.has_key(lower(changes["changedbyname"])): + if uploadernames.has_key(changes["changedbyname"].lower()): return 0; # Some group maintained packages (e.g. Debian QA) are never NMU's - if self.group_maint.has_key(lower(changes["maintaineremail"])): + if self.group_maint.has_key(changes["maintaineremail"].lower()): return 0; return 1; @@ -143,6 +142,16 @@ class Katie: exec "%s = self.pkg.%s;" % (i,i); dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".katie"); dump_file = utils.open_file(dump_filename, 'w'); + try: + os.chmod(dump_filename, 0660); + except OSError, e: + if errno.errorcode[e.errno] == 'EPERM': + perms = stat.S_IMODE(os.stat(dump_filename)[stat.ST_MODE]); + if perms & stat.S_IROTH: + utils.fubar("%s is world readable and chmod failed." % (dump_filename)); + else: + raise; + p = cPickle.Pickler(dump_file, 1); for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]: exec "%s = {}" % i; @@ -152,8 +161,8 @@ class Katie: for i in [ "package", "version", "architecture", "type", "size", "md5sum", "component", "location id", "source package", "source version", "maintainer", "dbtype", "files id", - "new", "section", "priority", "oldfiles", "othercomponents", - "pool name" ]: + "new", "section", "priority", "othercomponents", + "pool name", "original component" ]: if files[file].has_key(i): d_files[file][i] = files[file][i]; ## changes @@ -164,7 +173,8 @@ class Katie: d_changes[i] = changes[i]; # Optional changes fields # FIXME: changes should be mandatory - for i in [ "changed-by", "maintainer822", "filecontents", "format", "changes" ]: + for i in [ "changed-by", "maintainer822", "filecontents", "format", + "changes", "lisa note" ]: if changes.has_key(i): d_changes[i] = changes[i]; ## dsc @@ -201,7 +211,7 @@ class Katie: if not changes.has_key("maintainer822"): changes["maintainer822"] = self.Cnf["Dinstall::MyEmailAddress"]; - Subst["__ARCHITECTURE__"] = string.join(changes["architecture"].keys(), ' ' ); + Subst["__ARCHITECTURE__"] = " ".join(changes["architecture"].keys()); Subst["__CHANGES_FILENAME__"] = os.path.basename(self.pkg.changes_file); Subst["__FILE_CONTENTS__"] = changes.get("filecontents", ""); @@ -215,7 +225,7 @@ class Katie: Subst["__MAINTAINER_TO__"] = changes["maintainer822"]; Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown"); if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"): - Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"]) + Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"]) # Apply any global override of the Maintainer field if self.Cnf.get("Dinstall::OverrideMaintainer"): @@ -244,18 +254,18 @@ class Katie: for file in file_keys: if files[file].has_key("byhand"): byhand = 1 - summary = summary + file + " byhand\n" + summary += file + " byhand\n" elif files[file].has_key("new"): new = 1 - summary = summary + "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"]) + summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"]) if files[file].has_key("othercomponents"): - summary = summary + "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) + summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) if files[file]["type"] == "deb": - summary = summary + apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n'; + summary += apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(file)))["Description"] + '\n'; else: files[file]["pool name"] = utils.poolify (changes["source"], files[file]["component"]) destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file - summary = summary + file + "\n to " + destination + "\n" + summary += file + "\n to " + destination + "\n" short_summary = summary; @@ -263,9 +273,9 @@ class Katie: f = re_fdnic.sub("\n .\n", changes.get("changes","")); if byhand or new: - summary = summary + "Changes: " + f; + summary += "Changes: " + f; - summary = summary + self.announce(short_summary, 0) + summary += self.announce(short_summary, 0) return (summary, short_summary); @@ -283,9 +293,9 @@ class Katie: bugs.sort(); if not self.nmu.is_an_nmu(self.pkg): - summary = summary + "Closing bugs: "; + summary += "Closing bugs: "; for bug in bugs: - summary = summary + "%s " % (bug); + summary += "%s " % (bug); if action: Subst["__BUG_NUMBER__"] = bug; if changes["distribution"].has_key("stable"): @@ -302,18 +312,18 @@ distribution."""; if action: self.Logger.log(["closing bugs"]+bugs); else: # NMU - summary = summary + "Setting bugs to severity fixed: "; + summary += "Setting bugs to severity fixed: "; control_message = ""; for bug in bugs: - summary = summary + "%s " % (bug); - control_message = control_message + "tag %s + fixed\n" % (bug); + summary += "%s " % (bug); + control_message += "tag %s + fixed\n" % (bug); if action and control_message != "": Subst["__CONTROL_MESSAGE__"] = control_message; mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed"); utils.send_mail (mail_message, ""); if action: self.Logger.log(["setting bugs to fixed"]+bugs); - summary = summary + "\n"; + summary += "\n"; return summary; ########################################################################### @@ -322,7 +332,6 @@ distribution."""; Subst = self.Subst; Cnf = self.Cnf; changes = self.pkg.changes; - dsc = self.pkg.dsc; # Only do announcements for source uploads with a recent dpkg-dev installed if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"): @@ -337,7 +346,7 @@ distribution."""; if list == "" or lists_done.has_key(list): continue; lists_done[list] = 1; - summary = summary + "Announcing to %s\n" % (list); + summary += "Announcing to %s\n" % (list); if action: Subst["__ANNOUNCE_LIST_ADDRESS__"] = list; @@ -346,7 +355,7 @@ distribution."""; mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce"); utils.send_mail (mail_message, ""); - if Cnf.get("Dinstall::CloseBugs"): + if Cnf.FindB("Dinstall::CloseBugs"): summary = self.close_bugs(summary, action); return summary; @@ -368,8 +377,8 @@ distribution."""; file_keys = files.keys(); for file in file_keys: utils.move(file, Cnf["Dir::Queue::Accepted"]); - self.accept_bytes = self.accept_bytes + float(files[file]["size"]) - self.accept_count = self.accept_count + 1; + self.accept_bytes += float(files[file]["size"]) + self.accept_count += 1; # Send accept mail, announce to lists, close bugs and check for # override disparities @@ -381,24 +390,33 @@ distribution."""; self.announce(short_summary, 1) # Special support to enable clean auto-building of accepted packages - if Cnf.FindB("Dinstall::SpecialAcceptedAutoBuild") and \ - self.pkg.changes["distribution"].has_key("unstable"): - self.projectB.query("BEGIN WORK"); + self.projectB.query("BEGIN WORK"); + for suite in self.pkg.changes["distribution"].keys(): + if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"): + continue; + suite_id = db_access.get_suite_id(suite); + dest_dir = Cnf["Dir::AcceptedAutoBuild"]; + if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + dest_dir = os.path.join(dest_dir, suite); for file in file_keys: src = os.path.join(Cnf["Dir::Queue::Accepted"], file); - dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"], file); - # Create a symlink to it - os.symlink(src, dest); + dest = os.path.join(dest_dir, file); + if Cnf.FindB("Dinstall::SecurityAcceptedAutoBuild"): + # Copy it since the original won't be readable by www-data + utils.copy(src, dest); + else: + # Create a symlink to it + os.symlink(src, dest); # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest)); + self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest)); # If the .orig.tar.gz is in the pool, create a symlink to # it (if one doesn't already exist) if self.pkg.orig_tar_id: # Determine the .orig.tar.gz file name for dsc_file in self.pkg.dsc_files.keys(): - if dsc_file[-12:] == ".orig.tar.gz": + if dsc_file.endswith(".orig.tar.gz"): filename = dsc_file; - dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"],filename); + dest = os.path.join(dest_dir, filename); # If it doesn't exist, create a symlink if not os.path.exists(dest): # Find the .orig.tar.gz in the pool @@ -409,9 +427,12 @@ distribution."""; src = os.path.join(ql[0][0], ql[0][1]); os.symlink(src, dest); # Add it to the list of packages for later processing by apt-ftparchive - self.projectB.query("INSERT INTO unstable_accepted (filename, in_accepted) VALUES ('%s', 't')" % (dest)); + self.projectB.query("INSERT INTO accepted_autobuild (suite, filename, in_accepted) VALUES (%s, '%s', 't')" % (suite_id, dest)); + # if it does, update things to ensure it's not removed prematurely + else: + self.projectB.query("UPDATE accepted_autobuild SET in_accepted = 't', last_used = NULL WHERE filename = '%s' AND suite = %s" % (dest, suite_id)); - self.projectB.query("COMMIT WORK"); + self.projectB.query("COMMIT WORK"); ########################################################################### @@ -435,15 +456,15 @@ distribution."""; if not files[file].has_key("new") and files[file]["type"] == "deb": section = files[file]["section"]; override_section = files[file]["override section"]; - if lower(section) != lower(override_section) and section != "-": + if section.lower() != override_section.lower() and section != "-": # Ignore this; it's a common mistake and not worth whining about - if lower(section) == "non-us/main" and lower(override_section) == "non-us": + if section.lower() == "non-us/main" and override_section.lower() == "non-us": continue; - summary = summary + "%s: section is overridden from %s to %s.\n" % (file, section, override_section); + summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section); priority = files[file]["priority"]; override_priority = files[file]["override priority"]; if priority != override_priority and priority != "-": - summary = summary + "%s: priority is overridden from %s to %s.\n" % (file, priority, override_priority); + summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority); if summary == "": return; @@ -506,18 +527,18 @@ distribution."""; while answer == 'E': os.system("%s %s" % (editor, temp_filename)) file = utils.open_file(temp_filename); - reject_message = string.join(file.readlines()); + reject_message = " ".join(file.readlines()); file.close(); print "Reject message:"; print utils.prefix_multi_line_string(reject_message," "); prompt = "[R]eject, Edit, Abandon, Quit ?" answer = "XXX"; - while string.find(prompt, answer) == -1: + while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); m = re_default_answer.search(prompt); if answer == "": answer = m.group(1); - answer = string.upper(answer[:1]); + answer = answer[:1].upper(); os.unlink(temp_filename); if answer == 'A': return 1; @@ -624,7 +645,7 @@ distribution."""; type_id = db_access.get_override_type_id(type); # FIXME: nasty non-US speficic hack - if lower(component[:7]) == "non-us/": + if component[:7].lower() == "non-us/": component = component[7:]; q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" @@ -632,13 +653,14 @@ distribution."""; result = q.getresult(); # If checking for a source package fall back on the binary override type if type == "dsc" and not result: - type_id = db_access.get_override_type_id("deb"); - q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" - % (package, suite_id, component_id, type_id)); + deb_type_id = db_access.get_override_type_id("deb"); + udeb_type_id = db_access.get_override_type_id("udeb"); + q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id" + % (package, suite_id, component_id, deb_type_id, udeb_type_id)); result = q.getresult(); # Remember the section and priority so we can check them later if appropriate - if result != []: + if result: files[file]["override section"] = result[0][0]; files[file]["override priority"] = result[0][1]; @@ -651,24 +673,58 @@ distribution."""; # Unlike other rejects we add new lines first to avoid trailing # new lines when this message is passed back up to a caller. if self.reject_message: - self.reject_message = self.reject_message + "\n"; - self.reject_message = self.reject_message + prefix + str; + self.reject_message += "\n"; + self.reject_message += prefix + str; + + ################################################################################ + + def cross_suite_version_check(self, query_result, file, new_version): + """Ensure versions are newer than existing packages in target + suites and that cross-suite version checking rules as + set out in the conf file are satisfied.""" + + # Check versions for each target suite + for target_suite in self.pkg.changes["distribution"].keys(): + must_be_newer_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite))); + must_be_older_than = map(string.lower, self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite))); + # Enforce "must be newer than target suite" even if conffile omits it + if target_suite not in must_be_newer_than: + must_be_newer_than.append(target_suite); + for entry in query_result: + existent_version = entry[0]; + suite = entry[1]; + if suite in must_be_newer_than and \ + apt_pkg.VersionCompare(new_version, existent_version) != 1: + self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); + if suite in must_be_older_than and \ + apt_pkg.VersionCompare(new_version, existent_version) != -1: + self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); + + ################################################################################ - def check_binaries_against_db(self, file, suite): + def check_binary_against_db(self, file): self.reject_message = ""; files = self.pkg.files; - # Find any old binary packages - q = self.projectB.query("SELECT b.id, b.version, f.filename, l.path, c.name FROM binaries b, bin_associations ba, suite s, location l, component c, architecture a, files f WHERE b.package = '%s' AND s.suite_name = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') AND ba.bin = b.id AND ba.suite = s.id AND b.architecture = a.id AND f.location = l.id AND l.component = c.id AND b.file = f.id" - % (files[file]["package"], suite, files[file]["architecture"])) - for oldfile in q.dictresult(): - files[file]["oldfiles"][suite] = oldfile; - # Check versions [NB: per-suite only; no cross-suite checking done (yet)] - if apt_pkg.VersionCompare(files[file]["version"], oldfile["version"]) != 1: - self.reject("%s: old version (%s) >= new version (%s)." % (file, oldfile["version"], files[file]["version"])); + # Ensure version is sane + q = self.projectB.query(""" +SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su, + architecture a + WHERE b.package = '%s' AND (a.arch_string = '%s' OR a.arch_string = 'all') + AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id""" + % (files[file]["package"], + files[file]["architecture"])); + self.cross_suite_version_check(q.getresult(), file, files[file]["version"]); + # Check for any existing copies of the file - q = self.projectB.query("SELECT b.id FROM binaries b, architecture a WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' AND a.id = b.architecture" % (files[file]["package"], files[file]["version"], files[file]["architecture"])) - if q.getresult() != []: + q = self.projectB.query(""" +SELECT b.id FROM binaries b, architecture a + WHERE b.package = '%s' AND b.version = '%s' AND a.arch_string = '%s' + AND a.id = b.architecture""" + % (files[file]["package"], + files[file]["version"], + files[file]["architecture"])) + if q.getresult(): self.reject("can not overwrite existing copy of '%s' already in the archive." % (file)); return self.reject_message; @@ -676,20 +732,15 @@ distribution."""; ################################################################################ def check_source_against_db(self, file): - """Ensure source is newer than existing source in target suites.""" self.reject_message = ""; - changes = self.pkg.changes; dsc = self.pkg.dsc; - package = dsc.get("source"); - new_version = dsc.get("version"); - for suite in changes["distribution"].keys(): - q = self.projectB.query("SELECT s.version FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND su.suite_name = '%s' AND sa.source = s.id AND sa.suite = su.id" - % (package, suite)); - ql = map(lambda x: x[0], q.getresult()); - for old_version in ql: - if apt_pkg.VersionCompare(new_version, old_version) != 1: - self.reject("%s: Old version `%s' >= new version `%s'." % (file, old_version, new_version)); + # Ensure version is sane + q = self.projectB.query(""" +SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su + WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source"))); + self.cross_suite_version_check(q.getresult(), file, dsc.get("version")); + return self.reject_message; ################################################################################ @@ -726,7 +777,7 @@ distribution."""; if ql: # Ignore exact matches for .orig.tar.gz match = 0; - if dsc_file[-12:] == ".orig.tar.gz": + if dsc_file.endswith(".orig.tar.gz"): for i in ql: if files.has_key(dsc_file) and \ int(files[dsc_file]["size"]) == int(i[0]) and \ @@ -737,12 +788,12 @@ distribution."""; if not match: self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file)); - elif dsc_file[-12:] == ".orig.tar.gz": + elif dsc_file.endswith(".orig.tar.gz"): # Check in the pool q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); ql = q.getresult(); - if ql != []: + if ql: # Unfortunately, we make get more than one # match here if, for example, the package was # in potato but had a -sa upload in woody. So