X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=jennifer;h=715440433e0fc2c44e48af09d5ce3925faa77493;hb=924f122e42b40b043c7393ad4c9d523c5baacbb1;hp=222a4d9e0116d7e72aa6dd3e5bfce0a95fb2263e;hpb=aab7bc581c5c651507c3072a7606c7406577bf5b;p=dak.git diff --git a/jennifer b/jennifer index 222a4d9e..71544043 100755 --- a/jennifer +++ b/jennifer @@ -1,8 +1,8 @@ #!/usr/bin/env python # Checks Debian packages from Incoming -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: jennifer,v 1.50 2004-06-08 23:31:05 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: jennifer,v 1.62 2005-12-05 04:54:29 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,8 +37,6 @@ from types import *; ################################################################################ -re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); -re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$"); re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); @@ -47,7 +45,7 @@ re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.50 $"; +jennifer_version = "$Revision: 1.62 $"; Cnf = None; Options = None; @@ -182,11 +180,6 @@ def clean_holding(): def check_changes(): filename = pkg.changes_file; - # Defaults in case we bail out - changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; - changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; - changes["architecture"] = {}; - # Parse the .changes field into a dictionary try: changes.update(utils.parse_changes(filename)); @@ -208,7 +201,7 @@ def check_changes(): # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution", - "maintainer", "files", "changes"): + "maintainer", "files", "changes", "description"): if not changes.has_key(i): reject("%s: Missing mandatory field `%s'." % (filename, i)); return 0 # Avoid errors during later tests @@ -237,6 +230,9 @@ def check_changes(): changes["changedbyname"], changes["changedbyemail"]) = \ utils.fix_maintainer (changes.get("changed-by", "")); except utils.ParseMaintError, msg: + (changes["changedby822"], changes["changedby2047"], + changes["changedbyname"], changes["changedbyemail"]) = \ + ("", "", "", "") reject("%s: Changed-By field ('%s') failed to parse: %s" \ % (filename, changes["changed-by"], msg)); @@ -281,6 +277,9 @@ def check_distributions(): changes["distribution"][dest] = 1; if type != "silent-map": reject("Mapping %s to %s." % (source, dest),""); + if changes.has_key("distribution-version"): + if changes["distribution-version"].has_key(source): + changes["distribution-version"][source]=dest elif type == "map-unreleased": (source, dest) = args[1:3]; if changes["distribution"].has_key(source): @@ -295,6 +294,17 @@ def check_distributions(): if changes["distribution"].has_key(suite): del changes["distribution"][suite]; reject("Ignoring %s as a target suite." % (suite), "Warning: "); + elif type == "reject": + suite = args[1]; + if changes["distribution"].has_key(suite): + reject("Uploads to %s are not accepted." % (suite)); + elif type == "propup-version": + # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes" + # + # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'} + if changes["distribution"].has_key(args[1]): + changes.setdefault("distribution-version", {}) + for suite in args[2:]: changes["distribution-version"][suite]=suite # Ensure there is (still) a target distribution if changes["distribution"].keys() == []: @@ -307,6 +317,48 @@ def check_distributions(): ################################################################################ +def check_deb_ar(filename, control): + """Sanity check the ar of a .deb, i.e. that there is: + + o debian-binary + o control.tar.gz + o data.tar.gz or data.tar.bz2 + +in that order, and nothing else. If the third member is a +data.tar.bz2, an additional check is performed for the required +Pre-Depends on dpkg (>= 1.10.24).""" + cmd = "ar t %s" % (filename) + (result, output) = commands.getstatusoutput(cmd) + if result != 0: + reject("%s: 'ar t' invocation failed." % (filename)) + reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") + chunks = output.split('\n') + if len(chunks) != 3: + reject("%s: found %d chunks, expected 3." % (filename, len(chunks))) + if chunks[0] != "debian-binary": + reject("%s: first chunk is '%s', expected 'debian-binary'." % (filename, chunks[0])) + if chunks[1] != "control.tar.gz": + reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (filename, chunks[1])) + if chunks[2] == "data.tar.bz2": + # Packages using bzip2 compression must have a Pre-Depends on dpkg >= 1.10.24. + found_needed_predep = 0 + for parsed_dep in apt_pkg.ParseDepends(control.Find("Pre-Depends", "")): + for atom in parsed_dep: + (dep, version, constraint) = atom + if dep != "dpkg" or (constraint != ">=" and constraint != ">>") or \ + len(parsed_dep) > 1: # or'ed deps don't count + continue + if (constraint == ">=" and apt_pkg.VersionCompare(version, "1.10.24") < 0) or \ + (constraint == ">>" and apt_pkg.VersionCompare(version, "1.10.23") < 0): + continue + found_needed_predep = 1 + if not found_needed_predep: + reject("%s: uses bzip2 compression, but doesn't Pre-Depend on dpkg (>= 1.10.24)" % (filename)) + elif chunks[2] != "data.tar.gz": + reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (filename, chunks[2])) + +################################################################################ + def check_files(): global reprocess @@ -364,7 +416,7 @@ def check_files(): files[file]["type"] = "unreadable"; continue; # If it's byhand skip remaining checks - if files[file]["section"] == "byhand": + if files[file]["section"] == "byhand" or files[file]["section"] == "raw-installer": files[file]["byhand"] = 1; files[file]["type"] = "byhand"; # Checks for a binary package... @@ -486,6 +538,8 @@ def check_files(): # Check the version and for file overwrites reject(Katie.check_binary_against_db(file),""); + check_deb_ar(file, control) + # Checks for a source package... else: m = utils.re_issource.match(file); @@ -535,16 +589,13 @@ def check_files(): if files[file]["component"] == source: files[file]["original component"] = source; files[file]["component"] = dest; + # Ensure the component is valid for the target suite if Cnf.has_key("Suite:%s::Components" % (suite)) and \ files[file]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)): reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite)); continue; - # See if the package is NEW - if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): - files[file]["new"] = 1; - # Validate the component component = files[file]["component"]; component_id = db_access.get_component_id(component); @@ -552,6 +603,10 @@ def check_files(): reject("file '%s' has unknown component '%s'." % (file, component)); continue; + # See if the package is NEW + if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + files[file]["new"] = 1; + # Validate the priority if files[file]["priority"].find('/') != -1: reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])); @@ -615,11 +670,12 @@ def check_dsc(): # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: + reject("source uploads must contain a dsc file"); return 0; # Parse the .dsc file try: - dsc.update(utils.parse_changes(dsc_filename, dsc_whitespace_rules=1)); + dsc.update(utils.parse_changes(dsc_filename, signing_rules=1)); except utils.cant_open_exc: # if not -n copy_to_holding() will have done this for us... if Options["No-Action"]: @@ -660,7 +716,7 @@ def check_dsc(): utils.fix_maintainer (dsc["maintainer"]); except utils.ParseMaintError, msg: reject("%s: Maintainer field ('%s') failed to parse: %s" \ - % (dsc_filename, changes["changed-by"], msg)); + % (dsc_filename, dsc["maintainer"], msg)); # Validate the build-depends field(s) for field_name in [ "build-depends", "build-depends-indep" ]: @@ -915,7 +971,14 @@ def check_timestamps(): deb_file = utils.open_file(filename); apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz"); deb_file.seek(0); - apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz"); + try: + apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz") + except SystemError, e: + # If we can't find a data.tar.gz, look for data.tar.bz2 instead. + if not re.match(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)): + raise + deb_file.seek(0) + apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2") deb_file.close(); # future_files = tar.future_files.keys(); @@ -973,17 +1036,23 @@ def action (): (summary, short_summary) = Katie.build_summaries(); - byhand = new = ""; - for file in files.keys(): - if files[file].has_key("byhand"): - byhand = 1 - elif files[file].has_key("new"): - new = 1 + # q-unapproved hax0ring + queue_info = { + "New": { "is": is_new, "process": acknowledge_new }, + "Byhand" : { "is": is_byhand, "process": do_byhand }, + "Unembargo" : { "is": is_unembargo, "process": queue_unembargo }, + "Embargo" : { "is": is_embargo, "process": queue_embargo }, + } + queues = [ "New", "Byhand" ] + if Cnf.FindB("Dinstall::SecurityQueueHandling"): + queues += [ "Unembargo", "Embargo" ] (prompt, answer) = ("", "XXX") if Options["No-Action"] or Options["Automatic"]: answer = 'S' + queuekey = '' + if reject_message.find("Rejected") != -1: if upload_too_new(): print "SKIP (too new)\n" + reject_message,; @@ -993,21 +1062,29 @@ def action (): prompt = "[R]eject, Skip, Quit ?"; if Options["Automatic"]: answer = 'R'; - elif new: - print "NEW to %s\n%s%s" % (", ".join(changes["distribution"].keys()), reject_message, summary),; - prompt = "[N]ew, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'N'; - elif byhand: - print "BYHAND\n" + reject_message + summary,; - prompt = "[B]yhand, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'B'; else: - print "ACCEPT\n" + reject_message + summary,; - prompt = "[A]ccept, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'A'; + queue = None + for q in queues: + if queue_info[q]["is"](): + queue = q + break + if queue: + print "%s for %s\n%s%s" % ( + queue.upper(), ", ".join(changes["distribution"].keys()), + reject_message, summary), + queuekey = queue[0].upper() + if queuekey in "RQSA": + queuekey = "D" + prompt = "[D]ivert, Skip, Quit ?" + else: + prompt = "[%s]%s, Skip, Quit ?" % (queuekey, queue[1:].lower()) + if Options["Automatic"]: + answer = queuekey + else: + print "ACCEPT\n" + reject_message + summary,; + prompt = "[A]ccept, Skip, Quit ?"; + if Options["Automatic"]: + answer = 'A'; while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); @@ -1021,27 +1098,94 @@ def action (): Katie.do_reject(0, reject_message); elif answer == 'A': accept(summary, short_summary); - elif answer == 'B': - do_byhand(summary); - elif answer == 'N': - acknowledge_new (summary); + remove_from_unchecked() + elif answer == queuekey: + queue_info[queue]["process"](summary) + remove_from_unchecked() elif answer == 'Q': sys.exit(0) +def remove_from_unchecked(): + os.chdir (pkg.directory); + for file in files.keys(): + os.unlink(file); + os.unlink(pkg.changes_file); + ################################################################################ def accept (summary, short_summary): Katie.accept(summary, short_summary); Katie.check_override(); - # Finally, remove the originals from the unchecked directory - os.chdir (pkg.directory); - for file in files.keys(): - os.unlink(file); - os.unlink(pkg.changes_file); +################################################################################ + +def move_to_dir (dest, perms=0660, changesperms=0664): + utils.move (pkg.changes_file, dest, perms=changesperms); + for file in file_keys: + utils.move (file, dest, perms=perms); ################################################################################ +def is_unembargo (): + if os.getcwd() == Cnf["Dir::Queue::Disembargo"]: + if changes["architecture"].has_key("source"): + if Options["No-Action"]: return 1 + + Katie.projectB.query( + "INSERT INTO disembargo (package, version) VALUES ('%s', '%s')" % + (changes["package"], changes["version"])) + + q = Katie.projectB.query( + "SELECT package FROM disembargo WHERE package = '%s' AND version = '%s'" % + (changes["package"], changes["version"])) + ql = q.getresult() + if ql: + return 1 + return 0 + +def queue_unembargo (summary): + print "Moving to UNEMBARGOED holding area." + Logger.log(["Moving to unembargoed", pkg.changes_file]); + + Katie.dump_vars(Cnf["Dir::Queue::Unembargoed"]); + + file_keys = files.keys(); + + move_to_dir(Cnf["Dir::Queue::Unembargoed"]) + Katie.queue_build("unembargoed", Cnf["Dir::Queue::Unembargoed"]) + + # Check for override disparities + Katie.Subst["__SUMMARY__"] = summary; + Katie.check_override(); + +################################################################################ + +def is_embargo (): + return 0 + +def queue_embargo (summary): + print "Moving to EMBARGOED holding area." + Logger.log(["Moving to embargoed", pkg.changes_file]); + + Katie.dump_vars(Cnf["Dir::Queue::Embargoed"]); + + file_keys = files.keys(); + + move_to_dir(Cnf["Dir::Queue::Embargoed"]) + Katie.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) + + # Check for override disparities + Katie.Subst["__SUMMARY__"] = summary; + Katie.check_override(); + +################################################################################ + +def is_byhand (): + for file in files.keys(): + if files[file].has_key("byhand"): + return 1 + return 0 + def do_byhand (summary): print "Moving to BYHAND holding area." Logger.log(["Moving to byhand", pkg.changes_file]); @@ -1050,23 +1194,20 @@ def do_byhand (summary): file_keys = files.keys(); - # Move all the files into the byhand directory - utils.move (pkg.changes_file, Cnf["Dir::Queue::Byhand"]); - for file in file_keys: - utils.move (file, Cnf["Dir::Queue::Byhand"], perms=0660); + move_to_dir(Cnf["Dir::Queue::Byhand"]) # Check for override disparities Katie.Subst["__SUMMARY__"] = summary; Katie.check_override(); - # Finally remove the originals. - os.chdir (pkg.directory); - for file in file_keys: - os.unlink(file); - os.unlink(pkg.changes_file); - ################################################################################ +def is_new (): + for file in files.keys(): + if files[file].has_key("new"): + return 1 + return 0 + def acknowledge_new (summary): Subst = Katie.Subst; @@ -1077,10 +1218,7 @@ def acknowledge_new (summary): file_keys = files.keys(); - # Move all the files into the 'new' directory - utils.move (pkg.changes_file, Cnf["Dir::Queue::New"]); - for file in file_keys: - utils.move (file, Cnf["Dir::Queue::New"], perms=0660); + move_to_dir(Cnf["Dir::Queue::New"]) if not Options["No-Mail"]: print "Sending new ack."; @@ -1088,12 +1226,6 @@ def acknowledge_new (summary): new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.new"); utils.send_mail(new_ack_message); - # Finally remove the originals. - os.chdir (pkg.directory); - for file in file_keys: - os.unlink(file); - os.unlink(pkg.changes_file); - ################################################################################ # reprocess is necessary for the case of foo_1.2-1 and foo_1.2-2 in @@ -1111,6 +1243,9 @@ def process_it (changes_file): # Reset some globals reprocess = 1; Katie.init_vars(); + # Some defaults in case we can't fully process the .changes file + changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; + changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; reject_message = ""; # Absolutize the filename to avoid the requirement of being in the @@ -1131,7 +1266,10 @@ def process_it (changes_file): # rather than the original... pkg.changes_file = os.path.basename(pkg.changes_file); changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject); - valid_changes_p = check_changes(); + if changes["fingerprint"]: + valid_changes_p = check_changes(); + else: + valid_changes_p = 0; if valid_changes_p: while reprocess: check_distributions(); @@ -1176,7 +1314,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # Obtain lock if not in no-action mode and initialize the log