X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=jennifer;h=6349c23c084753c83a7dcb39717021d6d2d45f74;hb=80b0cf4b95f879ce7e6e41297bced5c4e955f697;hp=a3d93094cf77e696f7ffe92df3a736a8f385a573;hpb=b8f923796848479e35aafebb4fb7c7be1db46ef8;p=dak.git diff --git a/jennifer b/jennifer index a3d93094..6349c23c 100755 --- a/jennifer +++ b/jennifer @@ -1,8 +1,8 @@ #!/usr/bin/env python # Checks Debian packages from Incoming -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: jennifer,v 1.49 2004-04-20 00:08:27 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: jennifer,v 1.60 2005-11-25 09:35:09 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,8 +37,6 @@ from types import *; ################################################################################ -re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); -re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$"); re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); @@ -47,7 +45,7 @@ re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.49 $"; +jennifer_version = "$Revision: 1.60 $"; Cnf = None; Options = None; @@ -182,11 +180,6 @@ def clean_holding(): def check_changes(): filename = pkg.changes_file; - # Defaults in case we bail out - changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; - changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; - changes["architecture"] = {}; - # Parse the .changes field into a dictionary try: changes.update(utils.parse_changes(filename)); @@ -208,7 +201,7 @@ def check_changes(): # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution", - "maintainer", "files", "changes"): + "maintainer", "files", "changes", "description"): if not changes.has_key(i): reject("%s: Missing mandatory field `%s'." % (filename, i)); return 0 # Avoid errors during later tests @@ -237,6 +230,9 @@ def check_changes(): changes["changedbyname"], changes["changedbyemail"]) = \ utils.fix_maintainer (changes.get("changed-by", "")); except utils.ParseMaintError, msg: + (changes["changedby822"], changes["changedby2047"], + changes["changedbyname"], changes["changedbyemail"]) = \ + ("", "", "", "") reject("%s: Changed-By field ('%s') failed to parse: %s" \ % (filename, changes["changed-by"], msg)); @@ -281,6 +277,9 @@ def check_distributions(): changes["distribution"][dest] = 1; if type != "silent-map": reject("Mapping %s to %s." % (source, dest),""); + if changes.has_key("distribution-version"): + if changes["distribution-version"].has_key(source): + changes["distribution-version"][source]=dest elif type == "map-unreleased": (source, dest) = args[1:3]; if changes["distribution"].has_key(source): @@ -295,6 +294,17 @@ def check_distributions(): if changes["distribution"].has_key(suite): del changes["distribution"][suite]; reject("Ignoring %s as a target suite." % (suite), "Warning: "); + elif type == "reject": + suite = args[1]; + if changes["distribution"].has_key(suite): + reject("Uploads to %s are not accepted." % (suite)); + elif type == "propup-version": + # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes" + # + # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'} + if changes["distribution"].has_key(args[1]): + changes.setdefault("distribution-version", {}) + for suite in args[2:]: changes["distribution-version"][suite]=suite # Ensure there is (still) a target distribution if changes["distribution"].keys() == []: @@ -307,6 +317,48 @@ def check_distributions(): ################################################################################ +def check_deb_ar(filename, control): + """Sanity check the ar of a .deb, i.e. that there is: + + o debian-binary + o control.tar.gz + o data.tar.gz or data.tar.bz2 + +in that order, and nothing else. If the third member is a +data.tar.bz2, an additional check is performed for the required +Pre-Depends on dpkg (>= 1.10.24).""" + cmd = "ar t %s" % (filename) + (result, output) = commands.getstatusoutput(cmd) + if result != 0: + reject("%s: 'ar t' invocation failed." % (filename)) + reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") + chunks = output.split('\n') + if len(chunks) != 3: + reject("%s: found %d chunks, expected 3." % (filename, len(chunks))) + if chunks[0] != "debian-binary": + reject("%s: first chunk is '%s', expected 'debian-binary'." % (filename, chunks[0])) + if chunks[1] != "control.tar.gz": + reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (filename, chunks[1])) + if chunks[2] == "data.tar.bz2": + # Packages using bzip2 compression must have a Pre-Depends on dpkg >= 1.10.24. + found_needed_predep = 0 + for parsed_dep in apt_pkg.ParseDepends(control.Find("Pre-Depends", "")): + for atom in parsed_dep: + (dep, version, constraint) = atom + if dep != "dpkg" or (constraint != ">=" and constraint != ">>") or \ + len(parsed_dep) > 1: # or'ed deps don't count + continue + if (constraint == ">=" and apt_pkg.VersionCompare(version, "1.10.24") < 0) or \ + (constraint == ">>" and apt_pkg.VersionCompare(version, "1.10.23") < 0): + continue + found_needed_predep = 1 + if not found_needed_predep: + reject("%s: uses bzip2 compression, but doesn't Pre-Depend on dpkg (>= 1.10.24)" % (filename)) + elif chunks[2] != "data.tar.gz": + reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (filename, chunks[2])) + +################################################################################ + def check_files(): global reprocess @@ -364,7 +416,7 @@ def check_files(): files[file]["type"] = "unreadable"; continue; # If it's byhand skip remaining checks - if files[file]["section"] == "byhand": + if files[file]["section"] == "byhand" or files[file]["section"] == "raw-installer": files[file]["byhand"] = 1; files[file]["type"] = "byhand"; # Checks for a binary package... @@ -486,6 +538,8 @@ def check_files(): # Check the version and for file overwrites reject(Katie.check_binary_against_db(file),""); + check_deb_ar(file, control) + # Checks for a source package... else: m = utils.re_issource.match(file); @@ -535,16 +589,13 @@ def check_files(): if files[file]["component"] == source: files[file]["original component"] = source; files[file]["component"] = dest; + # Ensure the component is valid for the target suite if Cnf.has_key("Suite:%s::Components" % (suite)) and \ files[file]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)): reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite)); continue; - # See if the package is NEW - if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): - files[file]["new"] = 1; - # Validate the component component = files[file]["component"]; component_id = db_access.get_component_id(component); @@ -552,6 +603,10 @@ def check_files(): reject("file '%s' has unknown component '%s'." % (file, component)); continue; + # See if the package is NEW + if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + files[file]["new"] = 1; + # Validate the priority if files[file]["priority"].find('/') != -1: reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])); @@ -601,7 +656,7 @@ def check_dsc(): # Ensure there is source to check if not changes["architecture"].has_key("source"): - return; + return 1; # Find the .dsc dsc_filename = None; @@ -609,17 +664,18 @@ def check_dsc(): if files[file]["type"] == "dsc": if dsc_filename: reject("can not process a .changes file with multiple .dsc's."); - return; + return 0; else: dsc_filename = file; # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: - return; + reject("source uploads must contain a dsc file"); + return 0; # Parse the .dsc file try: - dsc.update(utils.parse_changes(dsc_filename, dsc_whitespace_rules=1)); + dsc.update(utils.parse_changes(dsc_filename, signing_rules=1)); except utils.cant_open_exc: # if not -n copy_to_holding() will have done this for us... if Options["No-Action"]: @@ -633,16 +689,16 @@ def check_dsc(): dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)); except utils.no_files_exc: reject("%s: no Files: field." % (dsc_filename)); - return; + return 0; except utils.changes_parse_error_exc, line: reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); - return; + return 0; # Enforce mandatory fields for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): if not dsc.has_key(i): reject("%s: missing mandatory field `%s'." % (dsc_filename, i)); - return; + return 0; # Validate the source and version fields if not re_valid_pkg_name.match(dsc["source"]): @@ -660,7 +716,7 @@ def check_dsc(): utils.fix_maintainer (dsc["maintainer"]); except utils.ParseMaintError, msg: reject("%s: Maintainer field ('%s') failed to parse: %s" \ - % (dsc_filename, changes["changed-by"], msg)); + % (dsc_filename, dsc["maintainer"], msg)); # Validate the build-depends field(s) for field_name in [ "build-depends", "build-depends-indep" ]: @@ -713,6 +769,8 @@ def check_dsc(): files[orig_tar_gz]["type"] = "orig.tar.gz"; reprocess = 2; + return 1; + ################################################################################ def get_changelog_versions(source_dir): @@ -913,7 +971,14 @@ def check_timestamps(): deb_file = utils.open_file(filename); apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz"); deb_file.seek(0); - apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz"); + try: + apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz") + except SystemError, e: + # If we can't find a data.tar.gz, look for data.tar.bz2 instead. + if not re.match(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)): + raise + deb_file.seek(0) + apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2") deb_file.close(); # future_files = tar.future_files.keys(); @@ -971,17 +1036,19 @@ def action (): (summary, short_summary) = Katie.build_summaries(); - byhand = new = ""; - for file in files.keys(): - if files[file].has_key("byhand"): - byhand = 1 - elif files[file].has_key("new"): - new = 1 + # q-unapproved hax0ring + queues = [ "New", "Byhand" ] + queue_info = { + "New": { "is": is_new, "process": acknowledge_new }, + "Byhand" : { "is": is_byhand, "process": do_byhand }, + } (prompt, answer) = ("", "XXX") if Options["No-Action"] or Options["Automatic"]: answer = 'S' + queuekey = '' + if reject_message.find("Rejected") != -1: if upload_too_new(): print "SKIP (too new)\n" + reject_message,; @@ -991,21 +1058,29 @@ def action (): prompt = "[R]eject, Skip, Quit ?"; if Options["Automatic"]: answer = 'R'; - elif new: - print "NEW to %s\n%s%s" % (", ".join(changes["distribution"].keys()), reject_message, summary),; - prompt = "[N]ew, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'N'; - elif byhand: - print "BYHAND\n" + reject_message + summary,; - prompt = "[B]yhand, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'B'; else: - print "ACCEPT\n" + reject_message + summary,; - prompt = "[A]ccept, Skip, Quit ?"; - if Options["Automatic"]: - answer = 'A'; + queue = None + for q in queues: + if queue_info[q]["is"](): + queue = q + break + if queue: + print "%s for %s\n%s%s" % ( + queue.upper(), ", ".join(changes["distribution"].keys()), + reject_message, summary), + queuekey = queue[0].upper() + if queuekey in "RQSA": + queuekey = "D" + prompt = "[D]ivert, Skip, Quit ?" + else: + prompt = "[%s]%s, Skip, Quit ?" % (queuekey, queue[1:].lower()) + if Options["Automatic"]: + answer = queuekey + else: + print "ACCEPT\n" + reject_message + summary,; + prompt = "[A]ccept, Skip, Quit ?"; + if Options["Automatic"]: + answer = 'A'; while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt); @@ -1019,10 +1094,8 @@ def action (): Katie.do_reject(0, reject_message); elif answer == 'A': accept(summary, short_summary); - elif answer == 'B': - do_byhand(summary); - elif answer == 'N': - acknowledge_new (summary); + elif answer == queuekey: + queue_info[queue]["process"](summary) elif answer == 'Q': sys.exit(0) @@ -1040,6 +1113,12 @@ def accept (summary, short_summary): ################################################################################ +def is_byhand (): + for file in files.keys(): + if files[file].has_key("byhand"): + return 1 + return 0 + def do_byhand (summary): print "Moving to BYHAND holding area." Logger.log(["Moving to byhand", pkg.changes_file]); @@ -1065,6 +1144,12 @@ def do_byhand (summary): ################################################################################ +def is_new (): + for file in files.keys(): + if files[file].has_key("new"): + return 1 + return 0 + def acknowledge_new (summary): Subst = Katie.Subst; @@ -1109,6 +1194,9 @@ def process_it (changes_file): # Reset some globals reprocess = 1; Katie.init_vars(); + # Some defaults in case we can't fully process the .changes file + changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; + changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; reject_message = ""; # Absolutize the filename to avoid the requirement of being in the @@ -1129,13 +1217,17 @@ def process_it (changes_file): # rather than the original... pkg.changes_file = os.path.basename(pkg.changes_file); changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject); - changes_valid = check_changes(); - if changes_valid: + if changes["fingerprint"]: + valid_changes_p = check_changes(); + else: + valid_changes_p = 0; + if valid_changes_p: while reprocess: check_distributions(); check_files(); - check_dsc(); - check_source(); + valid_dsc_p = check_dsc(); + if valid_dsc_p: + check_source(); check_md5sums(); check_urgency(); check_timestamps(); @@ -1173,7 +1265,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # Obtain lock if not in no-action mode and initialize the log