X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=jennifer;h=c19e1ec545a0b9a5d739b8a9a30d738f8ad2f45c;hb=950d3a6f5d3733d912a9e9b586d8de7d9405db7b;hp=c451a0538280ad911fe44c3bb2aeda9411b27c23;hpb=3686a00f1001f2d5692fa5e706b898053e39191a;p=dak.git diff --git a/jennifer b/jennifer index c451a053..c19e1ec5 100755 --- a/jennifer +++ b/jennifer @@ -1,8 +1,8 @@ #!/usr/bin/env python # Checks Debian packages from Incoming -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: jennifer,v 1.46 2004-04-01 17:14:25 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: jennifer,v 1.58 2005-11-24 15:40:16 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,8 +37,6 @@ from types import *; ################################################################################ -re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); -re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$"); re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); @@ -47,7 +45,7 @@ re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.46 $"; +jennifer_version = "$Revision: 1.58 $"; Cnf = None; Options = None; @@ -182,11 +180,6 @@ def clean_holding(): def check_changes(): filename = pkg.changes_file; - # Default in case we bail out - changes["maintainer822"] = Cnf["Dinstall::MyEmailAddress"]; - changes["changedby822"] = Cnf["Dinstall::MyEmailAddress"]; - changes["architecture"] = {}; - # Parse the .changes field into a dictionary try: changes.update(utils.parse_changes(filename)); @@ -207,7 +200,8 @@ def check_changes(): return 0; # Check for mandatory fields - for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"): + for i in ("source", "binary", "architecture", "version", "distribution", + "maintainer", "files", "changes", "description"): if not changes.has_key(i): reject("%s: Missing mandatory field `%s'." % (filename, i)); return 0 # Avoid errors during later tests @@ -221,11 +215,26 @@ def check_changes(): for j in o.split(): changes[i][j] = 1 - # Fix the Maintainer: field to be RFC822 compatible - (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) - - # Fix the Changed-By: field to be RFC822 compatible; if it exists. - (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by","")); + # Fix the Maintainer: field to be RFC822/2047 compatible + try: + (changes["maintainer822"], changes["maintainer2047"], + changes["maintainername"], changes["maintaineremail"]) = \ + utils.fix_maintainer (changes["maintainer"]); + except utils.ParseMaintError, msg: + reject("%s: Maintainer field ('%s') failed to parse: %s" \ + % (filename, changes["maintainer"], msg)); + + # ...likewise for the Changed-By: field if it exists. + try: + (changes["changedby822"], changes["changedby2047"], + changes["changedbyname"], changes["changedbyemail"]) = \ + utils.fix_maintainer (changes.get("changed-by", "")); + except utils.ParseMaintError, msg: + (changes["changedby822"], changes["changedby2047"], + changes["changedbyname"], changes["changedbyemail"]) = \ + ("", "", "", "") + reject("%s: Changed-By field ('%s') failed to parse: %s" \ + % (filename, changes["changed-by"], msg)); # Ensure all the values in Closes: are numbers if changes.has_key("closes"): @@ -268,6 +277,9 @@ def check_distributions(): changes["distribution"][dest] = 1; if type != "silent-map": reject("Mapping %s to %s." % (source, dest),""); + if changes.has_key("distribution-version"): + if changes["distribution-version"].has_key(source): + changes["distribution-version"][source]=dest elif type == "map-unreleased": (source, dest) = args[1:3]; if changes["distribution"].has_key(source): @@ -282,6 +294,17 @@ def check_distributions(): if changes["distribution"].has_key(suite): del changes["distribution"][suite]; reject("Ignoring %s as a target suite." % (suite), "Warning: "); + elif type == "reject": + suite = args[1]; + if changes["distribution"].has_key(suite): + reject("Uploads to %s are not accepted." % (suite)); + elif type == "propup-version": + # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes" + # + # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'} + if changes["distribution"].has_key(args[1]): + changes.setdefault("distribution-version", {}) + for suite in args[2:]: changes["distribution-version"][suite]=suite # Ensure there is (still) a target distribution if changes["distribution"].keys() == []: @@ -294,6 +317,48 @@ def check_distributions(): ################################################################################ +def check_deb_ar(filename, control): + """Sanity check the ar of a .deb, i.e. that there is: + + o debian-binary + o control.tar.gz + o data.tar.gz or data.tar.bz2 + +in that order, and nothing else. If the third member is a +data.tar.bz2, an additional check is performed for the required +Pre-Depends on dpkg (>= 1.10.24).""" + cmd = "ar t %s" % (filename) + (result, output) = commands.getstatusoutput(cmd) + if result != 0: + reject("%s: 'ar t' invocation failed." % (filename)) + reject(utils.prefix_multi_line_string(output, " [ar output:] "), "") + chunks = output.split('\n') + if len(chunks) != 3: + reject("%s: found %d chunks, expected 3." % (filename, len(chunks))) + if chunks[0] != "debian-binary": + reject("%s: first chunk is '%s', expected 'debian-binary'." % (filename, chunks[0])) + if chunks[1] != "control.tar.gz": + reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (filename, chunks[1])) + if chunks[2] == "data.tar.bz2": + # Packages using bzip2 compression must have a Pre-Depends on dpkg >= 1.10.24. + found_needed_predep = 0 + for parsed_dep in apt_pkg.ParseDepends(control.Find("Pre-Depends", "")): + for atom in parsed_dep: + (dep, version, constraint) = atom + if dep != "dpkg" or (constraint != ">=" and constraint != ">>") or \ + len(parsed_dep) > 1: # or'ed deps don't count + continue + if (constraint == ">=" and apt_pkg.VersionCompare(version, "1.10.24") < 0) or \ + (constraint == ">>" and apt_pkg.VersionCompare(version, "1.10.23") < 0): + continue + found_needed_predep = 1 + if not found_needed_predep: + reject("%s: uses bzip2 compression, but doesn't Pre-Depend on dpkg (>= 1.10.24)" % (filename)) + elif chunks[2] != "data.tar.gz": + reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (filename, chunks[2])) + +################################################################################ + def check_files(): global reprocess @@ -310,6 +375,24 @@ def check_files(): copy_to_holding(file); os.chdir(cwd); + # Check there isn't already a .changes or .katie file of the same name in + # the proposed-updates "CopyChanges" or "CopyKatie" storage directories. + # [NB: this check must be done post-suite mapping] + base_filename = os.path.basename(pkg.changes_file); + katie_filename = base_filename[:-8]+".katie" + for suite in changes["distribution"].keys(): + copychanges = "Suite::%s::CopyChanges" % (suite); + if Cnf.has_key(copychanges) and \ + os.path.exists(Cnf[copychanges]+"/"+base_filename): + reject("%s: a file with this name already exists in %s" \ + % (base_filename, Cnf[copychanges])); + + copykatie = "Suite::%s::CopyKatie" % (suite); + if Cnf.has_key(copykatie) and \ + os.path.exists(Cnf[copykatie]+"/"+katie_filename): + reject("%s: a file with this name already exists in %s" \ + % (katie_filename, Cnf[copykatie])); + reprocess = 0; has_binaries = 0; has_source = 0; @@ -333,7 +416,7 @@ def check_files(): files[file]["type"] = "unreadable"; continue; # If it's byhand skip remaining checks - if files[file]["section"] == "byhand": + if files[file]["section"] == "byhand" or files[file]["section"] == "raw-installer": files[file]["byhand"] = 1; files[file]["type"] = "byhand"; # Checks for a binary package... @@ -455,6 +538,8 @@ def check_files(): # Check the version and for file overwrites reject(Katie.check_binary_against_db(file),""); + check_deb_ar(file, control) + # Checks for a source package... else: m = utils.re_issource.match(file); @@ -504,16 +589,13 @@ def check_files(): if files[file]["component"] == source: files[file]["original component"] = source; files[file]["component"] = dest; + # Ensure the component is valid for the target suite if Cnf.has_key("Suite:%s::Components" % (suite)) and \ files[file]["component"] not in Cnf.ValueList("Suite::%s::Components" % (suite)): reject("unknown component `%s' for suite `%s'." % (files[file]["component"], suite)); continue; - # See if the package is NEW - if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): - files[file]["new"] = 1; - # Validate the component component = files[file]["component"]; component_id = db_access.get_component_id(component); @@ -521,6 +603,10 @@ def check_files(): reject("file '%s' has unknown component '%s'." % (file, component)); continue; + # See if the package is NEW + if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + files[file]["new"] = 1; + # Validate the priority if files[file]["priority"].find('/') != -1: reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])); @@ -570,7 +656,7 @@ def check_dsc(): # Ensure there is source to check if not changes["architecture"].has_key("source"): - return; + return 1; # Find the .dsc dsc_filename = None; @@ -578,17 +664,18 @@ def check_dsc(): if files[file]["type"] == "dsc": if dsc_filename: reject("can not process a .changes file with multiple .dsc's."); - return; + return 0; else: dsc_filename = file; # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: - return; + reject("source uploads must contain a dsc file"); + return 0; # Parse the .dsc file try: - dsc.update(utils.parse_changes(dsc_filename, dsc_whitespace_rules=1)); + dsc.update(utils.parse_changes(dsc_filename, signing_rules=1)); except utils.cant_open_exc: # if not -n copy_to_holding() will have done this for us... if Options["No-Action"]: @@ -602,20 +689,21 @@ def check_dsc(): dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)); except utils.no_files_exc: reject("%s: no Files: field." % (dsc_filename)); - return; + return 0; except utils.changes_parse_error_exc, line: reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); - return; + return 0; # Enforce mandatory fields for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): if not dsc.has_key(i): reject("%s: missing mandatory field `%s'." % (dsc_filename, i)); + return 0; # Validate the source and version fields - if dsc.has_key("source") and not re_valid_pkg_name.match(dsc["source"]): + if not re_valid_pkg_name.match(dsc["source"]): reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"])); - if dsc.has_key("version") and not re_valid_version.match(dsc["version"]): + if not re_valid_version.match(dsc["version"]): reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"])); # Bumping the version number of the .dsc breaks extraction by stable's @@ -623,6 +711,13 @@ def check_dsc(): if dsc["format"] != "1.0": reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename)); + # Validate the Maintainer field + try: + utils.fix_maintainer (dsc["maintainer"]); + except utils.ParseMaintError, msg: + reject("%s: Maintainer field ('%s') failed to parse: %s" \ + % (dsc_filename, dsc["maintainer"], msg)); + # Validate the build-depends field(s) for field_name in [ "build-depends", "build-depends-indep" ]: field = dsc.get(field_name); @@ -639,7 +734,7 @@ def check_dsc(): pass; # Ensure the version number in the .dsc matches the version number in the .changes - epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version")); + epochless_dsc_version = utils.re_no_epoch.sub('', dsc["version"]); changes_version = files[dsc_filename]["version"]; if epochless_dsc_version != files[dsc_filename]["version"]: reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version)); @@ -674,6 +769,8 @@ def check_dsc(): files[orig_tar_gz]["type"] = "orig.tar.gz"; reprocess = 2; + return 1; + ################################################################################ def get_changelog_versions(source_dir): @@ -701,8 +798,7 @@ def get_changelog_versions(source_dir): type = m.group(3); if type == "orig.tar.gz" and pkg.orig_tar_gz: continue; - else: - dest = os.path.join(os.getcwd(), f); + dest = os.path.join(os.getcwd(), f); os.symlink(src, dest); # If the orig.tar.gz is not a part of the upload, create a symlink to the @@ -774,7 +870,22 @@ def check_source(): # Move back and cleanup the temporary tree os.chdir(cwd); - shutil.rmtree(tmpdir); + try: + shutil.rmtree(tmpdir); + except OSError, e: + if errno.errorcode[e.errno] != 'EACCES': + utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])); + + reject("%s: source tree could not be cleanly removed." % (dsc["source"])); + # We probably have u-r or u-w directories so chmod everything + # and try again. + cmd = "chmod -R u+rwx %s" % (tmpdir) + result = os.system(cmd) + if result != 0: + utils.fubar("'%s' failed with result %s." % (cmd, result)); + shutil.rmtree(tmpdir); + except: + utils.fubar("%s: couldn't remove tmp dir for source tree." % (dsc["source"])); ################################################################################ @@ -860,7 +971,14 @@ def check_timestamps(): deb_file = utils.open_file(filename); apt_inst.debExtract(deb_file,tar.callback,"control.tar.gz"); deb_file.seek(0); - apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz"); + try: + apt_inst.debExtract(deb_file,tar.callback,"data.tar.gz") + except SystemError, e: + # If we can't find a data.tar.gz, look for data.tar.bz2 instead. + if not re.match(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)): + raise + deb_file.seek(0) + apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2") deb_file.close(); # future_files = tar.future_files.keys(); @@ -1056,6 +1174,9 @@ def process_it (changes_file): # Reset some globals reprocess = 1; Katie.init_vars(); + # Some defaults in case we can't fully process the .changes file + changes["maintainer2047"] = Cnf["Dinstall::MyEmailAddress"]; + changes["changedby2047"] = Cnf["Dinstall::MyEmailAddress"]; reject_message = ""; # Absolutize the filename to avoid the requirement of being in the @@ -1076,13 +1197,17 @@ def process_it (changes_file): # rather than the original... pkg.changes_file = os.path.basename(pkg.changes_file); changes["fingerprint"] = utils.check_signature(pkg.changes_file, reject); - changes_valid = check_changes(); - if changes_valid: + if changes["fingerprint"]: + valid_changes_p = check_changes(); + else: + valid_changes_p = 0; + if valid_changes_p: while reprocess: check_distributions(); check_files(); - check_dsc(); - check_source(); + valid_dsc_p = check_dsc(); + if valid_dsc_p: + check_source(); check_md5sums(); check_urgency(); check_timestamps(); @@ -1120,7 +1245,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # Obtain lock if not in no-action mode and initialize the log