X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=jennifer;h=c451a0538280ad911fe44c3bb2aeda9411b27c23;hb=3686a00f1001f2d5692fa5e706b898053e39191a;hp=132752588faa5c1125c5e39e92d445d7d588ae82;hpb=62c69fc3bb5fba75bc0ca305ed278246b3c5c974;p=dak.git diff --git a/jennifer b/jennifer index 13275258..c451a053 100755 --- a/jennifer +++ b/jennifer @@ -1,8 +1,8 @@ #!/usr/bin/env python # Checks Debian packages from Incoming -# Copyright (C) 2000, 2001, 2002, 2003 James Troup -# $Id: jennifer,v 1.39 2003-10-14 19:16:16 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup +# $Id: jennifer,v 1.46 2004-04-01 17:14:25 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ ################################################################################ -import errno, fcntl, gzip, os, re, shutil, stat, sys, time, traceback; +import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback; import apt_inst, apt_pkg; import db_access, katie, logging, utils; @@ -41,11 +41,13 @@ re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$"); re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); +re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); +re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.39 $"; +jennifer_version = "$Revision: 1.46 $"; Cnf = None; Options = None; @@ -137,7 +139,7 @@ def copy_to_holding(filename): # Shouldn't happen, but will if, for example, someone lists a # file twice in the .changes. if errno.errorcode[e.errno] == 'EEXIST': - reject("%s already exists in holding area; can not overwrite." % (base_filename)); + reject("%s: already exists in holding area; can not overwrite." % (base_filename)); return; raise; @@ -148,11 +150,11 @@ def copy_to_holding(filename): # O_CREAT | O_EXCLed ghost file, so add the file to the list # of 'in holding' even if it's not the real file. if errno.errorcode[e.errno] == 'ENOENT': - reject("can not copy %s to holding area: file not found." % (base_filename)); + reject("%s: can not copy to holding area: file not found." % (base_filename)); os.unlink(dest); return; elif errno.errorcode[e.errno] == 'EACCES': - reject("can not copy %s to holding area: read permission denied." % (base_filename)); + reject("%s: can not copy to holding area: read permission denied." % (base_filename)); os.unlink(dest); return; raise; @@ -189,25 +191,25 @@ def check_changes(): try: changes.update(utils.parse_changes(filename)); except utils.cant_open_exc: - reject("can't read changes file '%s'." % (filename)); + reject("%s: can't read file." % (filename)); return 0; except utils.changes_parse_error_exc, line: - reject("error parsing changes file '%s', can't grok: %s." % (filename, line)); + reject("%s: parse error, can't grok: %s." % (filename, line)); return 0; # Parse the Files field from the .changes into another dictionary try: files.update(utils.build_file_list(changes)); except utils.changes_parse_error_exc, line: - reject("error parsing changes file '%s', can't grok: %s." % (filename, line)); + reject("%s: parse error, can't grok: %s." % (filename, line)); except utils.nk_format_exc, format: - reject("unknown format '%s' of changes file '%s'." % (format, filename)); + reject("%s: unknown format '%s'." % (filename, format)); return 0; # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution", "maintainer", "files"): if not changes.has_key(i): - reject("Missing field `%s' in changes file." % (i)); + reject("%s: Missing mandatory field `%s'." % (filename, i)); return 0 # Avoid errors during later tests # Split multi-value fields into a lower-level dictionary @@ -229,7 +231,7 @@ def check_changes(): if changes.has_key("closes"): for i in changes["closes"].keys(): if katie.re_isanum.match (i) == None: - reject("`%s' from Closes field isn't a number." % (i)); + reject("%s: `%s' from Closes field isn't a number." % (filename, i)); # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison) @@ -243,6 +245,11 @@ def check_changes(): if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+base_filename): reject("%s: a file with this name already exists in the %s directory." % (base_filename, dir)); + # Check the .changes is non-empty + if not files: + reject("%s: nothing to do (Files field is empty)." % (base_filename)) + return 0; + return 1; ################################################################################ @@ -330,7 +337,7 @@ def check_files(): files[file]["byhand"] = 1; files[file]["type"] = "byhand"; # Checks for a binary package... - elif utils.re_isadeb.match(file) != None: + elif utils.re_isadeb.match(file): has_binaries = 1; files[file]["type"] = "deb"; @@ -383,9 +390,9 @@ def check_files(): reject("%s: Depends field is empty." % (file)); # Check the section & priority match those given in the .changes (non-fatal) - if control.Find("Section") != None and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"): + if control.Find("Section") and files[file]["section"] != "" and files[file]["section"] != control.Find("Section"): reject("%s control file lists section as `%s', but changes file has `%s'." % (file, control.Find("Section", ""), files[file]["section"]), "Warning: "); - if control.Find("Priority") != None and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"): + if control.Find("Priority") and files[file]["priority"] != "" and files[file]["priority"] != control.Find("Priority"): reject("%s control file lists priority as `%s', but changes file has `%s'." % (file, control.Find("Priority", ""), files[file]["priority"]),"Warning: "); files[file]["package"] = package; @@ -401,11 +408,11 @@ def check_files(): files[file]["source"] = control.Find("Source", files[file]["package"]); # Get the source version source = files[file]["source"]; - source_version = "" + source_version = ""; if source.find("(") != -1: - m = utils.re_extract_src_version.match(source) - source = m.group(1) - source_version = m.group(2) + m = utils.re_extract_src_version.match(source); + source = m.group(1); + source_version = m.group(2); if not source_version: source_version = files[file]["version"]; files[file]["source package"] = source; @@ -451,7 +458,7 @@ def check_files(): # Checks for a source package... else: m = utils.re_issource.match(file); - if m != None: + if m: has_source = 1; files[file]["package"] = m.group(1); files[file]["version"] = m.group(2); @@ -558,116 +565,216 @@ SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, ############################################################################### -def check_dsc (): +def check_dsc(): global reprocess; + # Ensure there is source to check + if not changes["architecture"].has_key("source"): + return; + + # Find the .dsc + dsc_filename = None; for file in files.keys(): - # The .orig.tar.gz can disappear out from under us is it's a - # duplicate of one in the archive. - if not files.has_key(file): - continue; if files[file]["type"] == "dsc": - # Parse the .dsc file - try: - dsc.update(utils.parse_changes(file, dsc_whitespace_rules=1)); - except utils.cant_open_exc: - # if not -n copy_to_holding() will have done this for us... - if Options["No-Action"]: - reject("can't read .dsc file '%s'." % (file)); - except utils.changes_parse_error_exc, line: - reject("error parsing .dsc file '%s', can't grok: %s." % (file, line)); - except utils.invalid_dsc_format_exc, line: - reject("syntax error in .dsc file '%s', line %s." % (file, line)); - # Build up the file list of files mentioned by the .dsc - try: - dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)); - except utils.no_files_exc: - reject("no Files: field in .dsc file."); - continue; - except utils.changes_parse_error_exc, line: - reject("error parsing .dsc file '%s', can't grok: %s." % (file, line)); - continue; + if dsc_filename: + reject("can not process a .changes file with multiple .dsc's."); + return; + else: + dsc_filename = file; + + # If there isn't one, we have nothing to do. (We have reject()ed the upload already) + if not dsc_filename: + return; - # Enforce mandatory fields - for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): - if not dsc.has_key(i): - reject("Missing field `%s' in dsc file." % (i)); - - # Validate the source and version fields - if dsc.has_key("source") and not re_valid_pkg_name.match(dsc["source"]): - reject("%s: invalid source name '%s'." % (file, dsc["source"])); - if dsc.has_key("version") and not re_valid_version.match(dsc["version"]): - reject("%s: invalid version number '%s'." % (file, dsc["version"])); - - # Bumping the version number of the .dsc breaks extraction by stable's - # dpkg-source. So let's not do that... - if dsc["format"] != "1.0": - reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (file)); - - # Validate the build-depends field(s) - for field_name in [ "build-depends", "build-depends-indep" ]: - field = dsc.get(field_name); - if field: - # Check for broken dpkg-dev lossage... - if field.find("ARRAY") == 0: - reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (file, field_name.title())); - - # Have apt try to parse them... - try: - apt_pkg.ParseSrcDepends(field); - except: - reject("%s: invalid %s field (can not be parsed by apt)." % (file, field_name.title())); - pass; - - # Ensure the version number in the .dsc matches the version number in the .changes - epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version")); - changes_version = files[file]["version"]; - if epochless_dsc_version != files[file]["version"]: - reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version)); - - # Ensure there is a .tar.gz in the .dsc file - has_tar = 0; - for f in dsc_files.keys(): - m = utils.re_issource.match(f); - if not m: - reject("%s mentioned in the Files field of %s not recognised as source." % (f, file)); - type = m.group(3); - if type == "orig.tar.gz" or type == "tar.gz": - has_tar = 1; - if not has_tar: - reject("no .tar.gz or .orig.tar.gz listed in the Files field of %s." % (file)); - - # Ensure source is newer than existing source in target suites - reject(Katie.check_source_against_db(file),""); - - (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); - reject(reject_msg, ""); - if is_in_incoming: - if not Options["No-Action"]: - copy_to_holding(is_in_incoming); - orig_tar_gz = os.path.basename(is_in_incoming); - files[orig_tar_gz] = {}; - files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE]; - files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"]; - files[orig_tar_gz]["section"] = files[file]["section"]; - files[orig_tar_gz]["priority"] = files[file]["priority"]; - files[orig_tar_gz]["component"] = files[file]["component"]; - files[orig_tar_gz]["type"] = "orig.tar.gz"; - reprocess = 2; + # Parse the .dsc file + try: + dsc.update(utils.parse_changes(dsc_filename, dsc_whitespace_rules=1)); + except utils.cant_open_exc: + # if not -n copy_to_holding() will have done this for us... + if Options["No-Action"]: + reject("%s: can't read file." % (dsc_filename)); + except utils.changes_parse_error_exc, line: + reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); + except utils.invalid_dsc_format_exc, line: + reject("%s: syntax error on line %s." % (dsc_filename, line)); + # Build up the file list of files mentioned by the .dsc + try: + dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)); + except utils.no_files_exc: + reject("%s: no Files: field." % (dsc_filename)); + return; + except utils.changes_parse_error_exc, line: + reject("%s: parse error, can't grok: %s." % (dsc_filename, line)); + return; + + # Enforce mandatory fields + for i in ("format", "source", "version", "binary", "maintainer", "architecture", "files"): + if not dsc.has_key(i): + reject("%s: missing mandatory field `%s'." % (dsc_filename, i)); + + # Validate the source and version fields + if dsc.has_key("source") and not re_valid_pkg_name.match(dsc["source"]): + reject("%s: invalid source name '%s'." % (dsc_filename, dsc["source"])); + if dsc.has_key("version") and not re_valid_version.match(dsc["version"]): + reject("%s: invalid version number '%s'." % (dsc_filename, dsc["version"])); + + # Bumping the version number of the .dsc breaks extraction by stable's + # dpkg-source. So let's not do that... + if dsc["format"] != "1.0": + reject("%s: incompatible 'Format' version produced by a broken version of dpkg-dev 1.9.1{3,4}." % (dsc_filename)); + + # Validate the build-depends field(s) + for field_name in [ "build-depends", "build-depends-indep" ]: + field = dsc.get(field_name); + if field: + # Check for broken dpkg-dev lossage... + if field.startswith("ARRAY"): + reject("%s: invalid %s field produced by a broken version of dpkg-dev (1.10.11)" % (dsc_filename, field_name.title())); + + # Have apt try to parse them... + try: + apt_pkg.ParseSrcDepends(field); + except: + reject("%s: invalid %s field (can not be parsed by apt)." % (dsc_filename, field_name.title())); + pass; + + # Ensure the version number in the .dsc matches the version number in the .changes + epochless_dsc_version = utils.re_no_epoch.sub('', dsc.get("version")); + changes_version = files[dsc_filename]["version"]; + if epochless_dsc_version != files[dsc_filename]["version"]: + reject("version ('%s') in .dsc does not match version ('%s') in .changes." % (epochless_dsc_version, changes_version)); + + # Ensure there is a .tar.gz in the .dsc file + has_tar = 0; + for f in dsc_files.keys(): + m = utils.re_issource.match(f); + if not m: + reject("%s: %s in Files field not recognised as source." % (dsc_filename, f)); + type = m.group(3); + if type == "orig.tar.gz" or type == "tar.gz": + has_tar = 1; + if not has_tar: + reject("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename)); + + # Ensure source is newer than existing source in target suites + reject(Katie.check_source_against_db(dsc_filename),""); + + (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(dsc_filename); + reject(reject_msg, ""); + if is_in_incoming: + if not Options["No-Action"]: + copy_to_holding(is_in_incoming); + orig_tar_gz = os.path.basename(is_in_incoming); + files[orig_tar_gz] = {}; + files[orig_tar_gz]["size"] = os.stat(orig_tar_gz)[stat.ST_SIZE]; + files[orig_tar_gz]["md5sum"] = dsc_files[orig_tar_gz]["md5sum"]; + files[orig_tar_gz]["section"] = files[dsc_filename]["section"]; + files[orig_tar_gz]["priority"] = files[dsc_filename]["priority"]; + files[orig_tar_gz]["component"] = files[dsc_filename]["component"]; + files[orig_tar_gz]["type"] = "orig.tar.gz"; + reprocess = 2; ################################################################################ -# dpkg-source broke .diff.gz generation in dpkg 1.8.x; detect the -# resulting bad source packages and reject them. +def get_changelog_versions(source_dir): + """Extracts a the source package and (optionally) grabs the + version history out of debian/changelog for the BTS.""" -def check_diff (): - for filename in files.keys(): - if files[filename]["type"] == "diff.gz": - file = gzip.GzipFile(filename, 'r'); - for line in file.readlines(): - if re_bad_diff.search(line): - reject("%s: invalid .diff.gz produced by a broken version of dpkg-dev 1.8.x." % (filename)); - break; + # Find the .dsc (again) + dsc_filename = None; + for file in files.keys(): + if files[file]["type"] == "dsc": + dsc_filename = file; + + # If there isn't one, we have nothing to do. (We have reject()ed the upload already) + if not dsc_filename: + return; + + # Create a symlink mirror of the source files in our temporary directory + for f in files.keys(): + m = utils.re_issource.match(f); + if m: + src = os.path.join(source_dir, f); + # If a file is missing for whatever reason, give up. + if not os.path.exists(src): + return; + type = m.group(3); + if type == "orig.tar.gz" and pkg.orig_tar_gz: + continue; + else: + dest = os.path.join(os.getcwd(), f); + os.symlink(src, dest); + + # If the orig.tar.gz is not a part of the upload, create a symlink to the + # existing copy. + if pkg.orig_tar_gz: + dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz)); + os.symlink(pkg.orig_tar_gz, dest); + + # Extract the source + cmd = "dpkg-source -sn -x %s" % (dsc_filename); + (result, output) = commands.getstatusoutput(cmd); + if (result != 0): + reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)); + reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), ""); + return; + + if not Cnf.Find("Dir::Queue::BTSVersionTrack"): + return; + + # Get the upstream version + upstr_version = utils.re_no_epoch.sub('', dsc["version"]); + if re_strip_revision.search(upstr_version): + upstr_version = re_strip_revision.sub('', upstr_version); + + # Ensure the changelog file exists + changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version); + if not os.path.exists(changelog_filename): + reject("%s: debian/changelog not found in extracted source." % (dsc_filename)); + return; + + # Parse the changelog + dsc["bts changelog"] = ""; + changelog_file = utils.open_file(changelog_filename); + for line in changelog_file.readlines(): + m = re_changelog_versions.match(line); + if m: + dsc["bts changelog"] += line; + changelog_file.close(); + + # Check we found at least one revision in the changelog + if not dsc["bts changelog"]: + reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename)); + +######################################## + +def check_source(): + # Bail out if: + # a) there's no source + # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files' + # or c) the orig.tar.gz is MIA + if not changes["architecture"].has_key("source") or reprocess == 2 \ + or pkg.orig_tar_gz == -1: + return; + + # Create a temporary directory to extract the source into + if Options["No-Action"]: + tmpdir = tempfile.mktemp(); + else: + # We're in queue/holding and can create a random directory. + tmpdir = "%s" % (os.getpid()); + os.mkdir(tmpdir); + + # Move into the temporary directory + cwd = os.getcwd(); + os.chdir(tmpdir); + + # Get the changelog version history + get_changelog_versions(cwd); + + # Move back and cleanup the temporary tree + os.chdir(cwd); + shutil.rmtree(tmpdir); ################################################################################ @@ -975,7 +1082,7 @@ def process_it (changes_file): check_distributions(); check_files(); check_dsc(); - check_diff(); + check_source(); check_md5sums(); check_urgency(); check_timestamps(); @@ -994,7 +1101,7 @@ def process_it (changes_file): ############################################################################### def main(): - global Cnf, Options, Logger, nmu; + global Cnf, Options, Logger; changes_files = init(); @@ -1054,7 +1161,7 @@ def main(): if accept_count: sets = "set" if accept_count > 1: - sets = "sets" + sets = "sets"; print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes))); Logger.log(["total",accept_count,accept_bytes]);