X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=40960b9040e297cd2c10953e5015389ba297caa4;hb=e79e9a1395528354b5978fc4b92a8fcdd1559d70;hp=f6bc3d4d104df702b8560d9bc2a19f4443074195;hpb=4e1aad646186f2df28c8dc8daeaabcdedbf161af;p=dak.git diff --git a/daklib/queue.py b/daklib/queue.py old mode 100644 new mode 100755 index f6bc3d4d..40960b90 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -22,6 +22,7 @@ import cPickle, errno, os, pg, re, stat, sys, time import apt_inst, apt_pkg import utils, database +from dak_exceptions import * from types import * @@ -32,6 +33,109 @@ re_default_answer = re.compile(r"\[(.*)\]") re_fdnic = re.compile(r"\n\n") re_bin_only_nmu = re.compile(r"\+b\d+$") +################################################################################ + +# Determine what parts in a .changes are NEW + +def determine_new(changes, files, projectB, warn=1): + new = {} + + # Build up a list of potentially new things + for file_entry in files.keys(): + f = files[file_entry] + # Skip byhand elements + if f["type"] == "byhand": + continue + pkg = f["package"] + priority = f["priority"] + section = f["section"] + file_type = get_type(f) + component = f["component"] + + if file_type == "dsc": + priority = "source" + if not new.has_key(pkg): + new[pkg] = {} + new[pkg]["priority"] = priority + new[pkg]["section"] = section + new[pkg]["type"] = file_type + new[pkg]["component"] = component + new[pkg]["files"] = [] + else: + old_type = new[pkg]["type"] + if old_type != file_type: + # source gets trumped by deb or udeb + if old_type == "dsc": + new[pkg]["priority"] = priority + new[pkg]["section"] = section + new[pkg]["type"] = file_type + new[pkg]["component"] = component + new[pkg]["files"].append(file_entry) + if f.has_key("othercomponents"): + new[pkg]["othercomponents"] = f["othercomponents"] + + for suite in changes["suite"].keys(): + suite_id = database.get_suite_id(suite) + for pkg in new.keys(): + component_id = database.get_component_id(new[pkg]["component"]) + type_id = database.get_override_type_id(new[pkg]["type"]) + q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id)) + ql = q.getresult() + if ql: + for file_entry in new[pkg]["files"]: + if files[file_entry].has_key("new"): + del files[file_entry]["new"] + del new[pkg] + + if warn: + if changes["suite"].has_key("stable"): + print "WARNING: overrides will be added for stable!" + if changes["suite"].has_key("oldstable"): + print "WARNING: overrides will be added for OLDstable!" + for pkg in new.keys(): + if new[pkg].has_key("othercomponents"): + print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"]) + + return new + +################################################################################ + +def get_type(f): + # Determine the type + if f.has_key("dbtype"): + file_type = f["dbtype"] + elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]: + file_type = "dsc" + else: + utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type)) + + # Validate the override type + type_id = database.get_override_type_id(file_type) + if type_id == -1: + utils.fubar("invalid type (%s) for new. Say wha?" % (file_type)) + + return file_type + +################################################################################ + +# check if section/priority values are valid + +def check_valid(new): + for pkg in new.keys(): + section = new[pkg]["section"] + priority = new[pkg]["priority"] + file_type = new[pkg]["type"] + new[pkg]["section id"] = database.get_section_id(section) + new[pkg]["priority id"] = database.get_priority_id(new[pkg]["priority"]) + # Sanity checks + di = section.find("debian-installer") != -1 + if (di and file_type not in ("udeb", "dsc")) or (not di and file_type == "udeb"): + new[pkg]["section id"] = -1 + if (priority == "source" and file_type != "dsc") or \ + (priority != "source" and file_type == "dsc"): + new[pkg]["priority id"] = -1 + + ############################################################################### # Convenience wrapper to carry around all the package information in @@ -67,8 +171,11 @@ class Upload: ########################################################################### def init_vars (self): - for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]: - exec "self.pkg.%s.clear();" % (i) + self.pkg.changes.clear() + self.pkg.dsc.clear() + self.pkg.files.clear() + self.pkg.dsc_files.clear() + self.pkg.legacy_source_untouchable.clear() self.pkg.orig_tar_id = None self.pkg.orig_tar_location = "" self.pkg.orig_tar_gz = None @@ -79,10 +186,16 @@ class Upload: dump_filename = self.pkg.changes_file[:-8]+".dak" dump_file = utils.open_file(dump_filename) p = cPickle.Unpickler(dump_file) - for i in [ "changes", "dsc", "files", "dsc_files", "legacy_source_untouchable" ]: - exec "self.pkg.%s.update(p.load());" % (i) - for i in [ "orig_tar_id", "orig_tar_location" ]: - exec "self.pkg.%s = p.load();" % (i) + + self.pkg.changes.update(p.load()) + self.pkg.dsc.update(p.load()) + self.pkg.files.update(p.load()) + self.pkg.dsc_files.update(p.load()) + self.pkg.legacy_source_untouchable.update(p.load()) + + self.pkg.orig_tar_id = p.load() + self.pkg.orig_tar_location = p.load() + dump_file.close() ########################################################################### @@ -92,9 +205,15 @@ class Upload: # process-new use from process-unchecked def dump_vars(self, dest_dir): - for i in [ "changes", "dsc", "files", "dsc_files", - "legacy_source_untouchable", "orig_tar_id", "orig_tar_location" ]: - exec "%s = self.pkg.%s;" % (i,i) + + changes = self.pkg.changes + dsc = self.pkg.dsc + files = self.pkg.files + dsc_files = self.pkg.dsc_files + legacy_source_untouchable = self.pkg.legacy_source_untouchable + orig_tar_id = self.pkg.orig_tar_id + orig_tar_location = self.pkg.orig_tar_location + dump_filename = os.path.join(dest_dir,self.pkg.changes_file[:-8] + ".dak") dump_file = utils.open_file(dump_filename, 'w') try: @@ -108,18 +227,22 @@ class Upload: raise p = cPickle.Pickler(dump_file, 1) - for i in [ "d_changes", "d_dsc", "d_files", "d_dsc_files" ]: - exec "%s = {}" % i + d_changes = {} + d_dsc = {} + d_files = {} + d_dsc_files = {} + ## files - for file in files.keys(): - d_files[file] = {} + for file_entry in files.keys(): + d_files[file_entry] = {} for i in [ "package", "version", "architecture", "type", "size", - "md5sum", "component", "location id", "source package", - "source version", "maintainer", "dbtype", "files id", - "new", "section", "priority", "othercomponents", + "md5sum", "sha1sum", "sha256sum", "component", + "location id", "source package", "source version", + "maintainer", "dbtype", "files id", "new", + "section", "priority", "othercomponents", "pool name", "original component" ]: - if files[file].has_key(i): - d_files[file][i] = files[file][i] + if files[file_entry].has_key(i): + d_files[file_entry][i] = files[file_entry][i] ## changes # Mandatory changes fields for i in [ "distribution", "source", "architecture", "version", @@ -129,7 +252,8 @@ class Upload: "closes", "changes" ]: d_changes[i] = changes[i] # Optional changes fields - for i in [ "changed-by", "filecontents", "format", "process-new note", "adv id", "distribution-version" ]: + for i in [ "changed-by", "filecontents", "format", "process-new note", "adv id", "distribution-version", + "sponsoremail" ]: if changes.has_key(i): d_changes[i] = changes[i] ## dsc @@ -138,15 +262,15 @@ class Upload: if dsc.has_key(i): d_dsc[i] = dsc[i] ## dsc_files - for file in dsc_files.keys(): - d_dsc_files[file] = {} + for file_entry in dsc_files.keys(): + d_dsc_files[file_entry] = {} # Mandatory dsc_files fields for i in [ "size", "md5sum" ]: - d_dsc_files[file][i] = dsc_files[file][i] + d_dsc_files[file_entry][i] = dsc_files[file_entry][i] # Optional dsc_files fields for i in [ "files id" ]: - if dsc_files[file].has_key(i): - d_dsc_files[file][i] = dsc_files[file][i] + if dsc_files[file_entry].has_key(i): + d_dsc_files[file_entry][i] = dsc_files[file_entry][i] for i in [ d_changes, d_dsc, d_files, d_dsc_files, legacy_source_untouchable, orig_tar_id, orig_tar_location ]: @@ -181,6 +305,10 @@ class Upload: Subst["__MAINTAINER_FROM__"] = changes["maintainer2047"] Subst["__MAINTAINER_TO__"] = changes["maintainer2047"] Subst["__MAINTAINER__"] = changes.get("maintainer", "Unknown") + + if "sponsoremail" in changes: + Subst["__MAINTAINER_TO__"] += ", %s"%changes["sponsoremail"] + if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"): Subst["__MAINTAINER_TO__"] += "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"]) @@ -209,31 +337,31 @@ class Upload: override_summary =""; file_keys = files.keys() file_keys.sort() - for file in file_keys: - if files[file].has_key("byhand"): + for file_entry in file_keys: + if files[file_entry].has_key("byhand"): byhand = 1 - summary += file + " byhand\n" - elif files[file].has_key("new"): + summary += file_entry + " byhand\n" + elif files[file_entry].has_key("new"): new = 1 - summary += "(new) %s %s %s\n" % (file, files[file]["priority"], files[file]["section"]) - if files[file].has_key("othercomponents"): - summary += "WARNING: Already present in %s distribution.\n" % (files[file]["othercomponents"]) - if files[file]["type"] == "deb": - deb_fh = utils.open_file(file) + summary += "(new) %s %s %s\n" % (file_entry, files[file_entry]["priority"], files[file_entry]["section"]) + if files[file_entry].has_key("othercomponents"): + summary += "WARNING: Already present in %s distribution.\n" % (files[file_entry]["othercomponents"]) + if files[file_entry]["type"] == "deb": + deb_fh = utils.open_file(file_entry) summary += apt_pkg.ParseSection(apt_inst.debExtractControl(deb_fh))["Description"] + '\n' deb_fh.close() else: - files[file]["pool name"] = utils.poolify (changes.get("source",""), files[file]["component"]) - destination = self.Cnf["Dir::PoolRoot"] + files[file]["pool name"] + file - summary += file + "\n to " + destination + "\n" - if not files[file].has_key("type"): - files[file]["type"] = "unknown" - if files[file]["type"] in ["deb", "udeb", "dsc"]: + files[file_entry]["pool name"] = utils.poolify (changes.get("source",""), files[file_entry]["component"]) + destination = self.Cnf["Dir::PoolRoot"] + files[file_entry]["pool name"] + file_entry + summary += file_entry + "\n to " + destination + "\n" + if not files[file_entry].has_key("type"): + files[file_entry]["type"] = "unknown" + if files[file_entry]["type"] in ["deb", "udeb", "dsc"]: # (queue/unchecked), there we have override entries already, use them # (process-new), there we dont have override entries, use the newly generated ones. - override_prio = files[file].get("override priority", files[file]["priority"]) - override_sect = files[file].get("override section", files[file]["section"]) - override_summary += "%s - %s %s\n" % (file, override_prio, override_sect) + override_prio = files[file_entry].get("override priority", files[file_entry]["priority"]) + override_sect = files[file_entry].get("override section", files[file_entry]["section"]) + override_summary += "%s - %s %s\n" % (file_entry, override_prio, override_sect) short_summary = summary @@ -300,14 +428,14 @@ distribution.""" Subst["__SHORT_SUMMARY__"] = short_summary for dist in changes["distribution"].keys(): - list = Cnf.Find("Suite::%s::Announce" % (dist)) - if list == "" or lists_done.has_key(list): + announce_list = Cnf.Find("Suite::%s::Announce" % (dist)) + if announce_list == "" or lists_done.has_key(announce_list): continue - lists_done[list] = 1 - summary += "Announcing to %s\n" % (list) + lists_done[announce_list] = 1 + summary += "Announcing to %s\n" % (announce_list) if action: - Subst["__ANNOUNCE_LIST_ADDRESS__"] = list + Subst["__ANNOUNCE_LIST_ADDRESS__"] = announce_list if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"): Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]) mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/process-unchecked.announce") @@ -336,9 +464,9 @@ distribution.""" # Move all the files into the accepted directory utils.move(changes_file, Cnf["Dir::Queue::Accepted"]) file_keys = files.keys() - for file in file_keys: - utils.move(file, Cnf["Dir::Queue::Accepted"]) - self.accept_bytes += float(files[file]["size"]) + for file_entry in file_keys: + utils.move(file_entry, Cnf["Dir::Queue::Accepted"]) + self.accept_bytes += float(files[file_entry]["size"]) self.accept_count += 1 # Send accept mail, announce to lists, close bugs and check for @@ -374,8 +502,8 @@ distribution.""" temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], dotprefix=1, perms=0644) debinfo = utils.open_file(temp_filename, 'w') - for file in file_keys: - f = files[file] + for file_entry in file_keys: + f = files[file_entry] if f["type"] == "deb": line = " ".join([f["package"], f["version"], f["architecture"], f["source package"], @@ -410,9 +538,9 @@ distribution.""" dest_dir = Cnf["Dir::QueueBuild"] if Cnf.FindB("Dinstall::SecurityQueueBuild"): dest_dir = os.path.join(dest_dir, suite) - for file in file_keys: - src = os.path.join(path, file) - dest = os.path.join(dest_dir, file) + for file_entry in file_keys: + src = os.path.join(path, file_entry) + dest = os.path.join(dest_dir, file_entry) if Cnf.FindB("Dinstall::SecurityQueueBuild"): # Copy it since the original won't be readable by www-data utils.copy(src, dest) @@ -466,19 +594,16 @@ distribution.""" summary = "" file_keys = files.keys() file_keys.sort() - for file in file_keys: - if not files[file].has_key("new") and files[file]["type"] == "deb": - section = files[file]["section"] - override_section = files[file]["override section"] + for file_entry in file_keys: + if not files[file_entry].has_key("new") and files[file_entry]["type"] == "deb": + section = files[file_entry]["section"] + override_section = files[file_entry]["override section"] if section.lower() != override_section.lower() and section != "-": - # Ignore this; it's a common mistake and not worth whining about - if section.lower() == "non-us/main" and override_section.lower() == "non-us": - continue - summary += "%s: package says section is %s, override says %s.\n" % (file, section, override_section) - priority = files[file]["priority"] - override_priority = files[file]["override priority"] + summary += "%s: package says section is %s, override says %s.\n" % (file_entry, section, override_section) + priority = files[file_entry]["priority"] + override_priority = files[file_entry]["override priority"] if priority != override_priority and priority != "-": - summary += "%s: package says priority is %s, override says %s.\n" % (file, priority, override_priority) + summary += "%s: package says priority is %s, override says %s.\n" % (file_entry, priority, override_priority) if summary == "": return @@ -497,36 +622,36 @@ distribution.""" Cnf = self.Cnf - for file in files: + for file_entry in files: # Skip any files which don't exist or which we don't have permission to copy. - if os.access(file,os.R_OK) == 0: + if os.access(file_entry,os.R_OK) == 0: continue - dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file) + dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file_entry) try: dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) except OSError, e: # File exists? Let's try and move it to the morgue if errno.errorcode[e.errno] == 'EEXIST': - morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file) + morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file_entry) try: morgue_file = utils.find_next_free(morgue_file) - except utils.tried_too_hard_exc: + except NoFreeFilenameError: # Something's either gone badly Pete Tong, or # someone is trying to exploit us. - utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file)) + utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry)) return utils.move(dest_file, morgue_file, perms=0660) try: dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644) except OSError, e: # Likewise - utils.warn("**WARNING** failed to claim %s in the reject directory." % (file)) + utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry)) return else: raise # If we got here, we own the destination file, so we can # safely overwrite it. - utils.move(file, dest_file, 1, perms=0660) + utils.move(file_entry, dest_file, 1, perms=0660) os.close(dest_fd) ########################################################################### @@ -613,26 +738,26 @@ distribution.""" # (2) Bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 def source_exists (self, package, source_version, suites = ["any"]): - okay = 1 - for suite in suites: - if suite == "any": - que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \ - (package) - else: - # source must exist in suite X, or in some other suite that's - # mapped to X, recursively... silent-maps are counted too, - # unreleased-maps aren't. - maps = self.Cnf.ValueList("SuiteMappings")[:] - maps.reverse() - maps = [ m.split() for m in maps ] - maps = [ (x[1], x[2]) for x in maps - if x[0] == "map" or x[0] == "silent-map" ] - s = [suite] - for x in maps: - if x[1] in s and x[0] not in s: - s.append(x[0]) - - que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, " OR ".join(["su.suite_name = '%s'" % a for a in s])) + okay = 1 + for suite in suites: + if suite == "any": + que = "SELECT s.version FROM source s WHERE s.source = '%s'" % \ + (package) + else: + # source must exist in suite X, or in some other suite that's + # mapped to X, recursively... silent-maps are counted too, + # unreleased-maps aren't. + maps = self.Cnf.ValueList("SuiteMappings")[:] + maps.reverse() + maps = [ m.split() for m in maps ] + maps = [ (x[1], x[2]) for x in maps + if x[0] == "map" or x[0] == "silent-map" ] + s = [suite] + for x in maps: + if x[1] in s and x[0] not in s: + s.append(x[0]) + + que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, " OR ".join(["su.suite_name = '%s'" % a for a in s])) q = self.projectB.query(que) # Reduce the query results to a list of version numbers @@ -649,18 +774,18 @@ distribution.""" # No source found... okay = 0 - break - return okay + break + return okay ################################################################################ - + def in_override_p (self, package, component, suite, binary_type, file): files = self.pkg.files if binary_type == "": # must be source - type = "dsc" + file_type = "dsc" else: - type = binary_type + file_type = binary_type # Override suite name; used for example with proposed-updates if self.Cnf.Find("Suite::%s::OverrideSuite" % (suite)) != "": @@ -671,17 +796,13 @@ distribution.""" if suite_id == -1: return None component_id = database.get_component_id(component) - type_id = database.get_override_type_id(type) - - # FIXME: nasty non-US speficic hack - if component.lower().startswith("non-us/"): - component = component[7:] + type_id = database.get_override_type_id(file_type) q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND type = %s AND o.section = s.id AND o.priority = p.id" % (package, suite_id, component_id, type_id)) result = q.getresult() # If checking for a source package fall back on the binary override type - if type == "dsc" and not result: + if file_type == "dsc" and not result: deb_type_id = database.get_override_type_id("deb") udeb_type_id = database.get_override_type_id("udeb") q = self.projectB.query("SELECT s.section, p.priority FROM override o, section s, priority p WHERE package = '%s' AND suite = %s AND component = %s AND (type = %s OR type = %s) AND o.section = s.id AND o.priority = p.id" @@ -741,12 +862,12 @@ distribution.""" ch = self.pkg.changes cansave = 0 if ch.get('distribution-version', {}).has_key(suite): - # we really use the other suite, ignoring the conflicting one ... + # we really use the other suite, ignoring the conflicting one ... addsuite = ch["distribution-version"][suite] - + add_version = self.get_anyversion(query_result, addsuite) target_version = self.get_anyversion(query_result, target_suite) - + if not add_version: # not add_version can only happen if we map to a suite # that doesn't enhance the suite we're propup'ing from. @@ -777,7 +898,7 @@ distribution.""" self.pkg.changes.setdefault("propdistribution", {}) self.pkg.changes["propdistribution"][addsuite] = 1 cansave = 1 - + if not cansave: self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)) @@ -932,13 +1053,13 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file) # See process_it() in 'dak process-unchecked' for explanation of this - # in_unchecked check dropped by ajt 2007-08-28, how did that - # ever make sense? + # in_unchecked check dropped by ajt 2007-08-28, how did that + # ever make sense? if os.path.exists(in_unchecked) and False: return (self.reject_message, in_unchecked) else: - for dir in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates" ]: - in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file) + for directory in [ "Accepted", "New", "Byhand", "ProposedUpdates", "OldProposedUpdates" ]: + in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (directory)],dsc_file) if os.path.exists(in_otherdir): in_otherdir_fh = utils.open_file(in_otherdir) actual_md5 = apt_pkg.md5sum(in_otherdir_fh)