From: James Troup Date: Thu, 1 Apr 2004 17:14:25 +0000 (+0000) Subject: 2004-04-01 James Troup * jennifer (get_changelog_versions):... X-Git-Url: https://git.decadent.org.uk/gitweb/?a=commitdiff_plain;h=3686a00f1001f2d5692fa5e706b898053e39191a;p=dak.git 2004-04-01 James Troup * jennifer (get_changelog_versions): create a symlink mirror of the source files in the temporary directory. (check_source): if check_dsc_against_db() couldn't find the orig.tar.gz bail out. * katie.py (Katie.check_dsc_against_db): if the orig.tar.gz is not part of the upload store the path to it in pkg.orig_tar_gz and if it can't be found set pkg.orig_tar_gz to -1. Explicitly return the second value as None in the (usual) case where we don't have to reprocess. Remove obsolete diagnostic logs. * katie.py (Katie.do_reject): use utils.temp_file() and don't import tempfile.2004-03-15 James Troup * jennifer (check_dsc): check changes["architecture"] for source before we do anything else.2004-03-12 James Troup * docs/README.config (Dir::Queue::BTSVersionTrack): document. * katie.conf (Dir::Queue::BTSVersionTrack): define. * katie.py (Katie.accept): add support for DebBugs Version Tracking by writing out .versions (generated in jennifer's get_changelog_versions()) and .debinfo (mapping of binary -> source) files. * ashley (main): add dsc["bts changelog"]. * katie.py (Katie.dump_vars): store dsc["bts changelog"] too. * jennifer (check_diff): obsoleted by check_source(), removed. (check_source): new function: create a temporary directory and move into it and call get_changelog_versions(). (get_changelog_versions): new function: extract the source package and optionally parse debian/changelog to obtain the version history for the BTS. (process_it): call check_source() rather than check_diff(). --- diff --git a/TODO b/TODO index c096c009..6d94b1d4 100644 --- a/TODO +++ b/TODO @@ -616,3 +616,19 @@ http://slashdot.org/comments.pl?sid=97501&cid=8334726 | build faster than I and others can fix it. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +(Note that the above is a gross oversimplification, and ignores issues +including but not necessarily limited to subarchitectures, and quality +of hardware coverage within certian architectures. It contains forward +looking statements, and may cause cancer in lab animals.) + +Joey Hess in <20040317065216.GA29816@kitenet.net> + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + now there's a thought + DD trading cards + official joeyh action figure, with rapid-fire upload action + lamont with pump-action NMU flame-thrower! + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/ashley b/ashley index 756ea5eb..194b715d 100755 --- a/ashley +++ b/ashley @@ -1,8 +1,8 @@ #!/usr/bin/env python # Dump variables from a .katie file to stdout -# Copyright (C) 2001, 2002 James Troup -# $Id: ashley,v 1.8 2003-01-02 18:10:02 troup Exp $ +# Copyright (C) 2001, 2002, 2004 James Troup +# $Id: ashley,v 1.9 2004-04-01 17:14:25 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -85,7 +85,8 @@ def main(): dsc = k.pkg.dsc; print " Dsc:"; - for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]: + for i in [ "source", "version", "maintainer", "fingerprint", "uploaders", + "bts changelog" ]: if dsc.has_key(i): print " %s: %s" % (i.capitalize(), dsc[i]); del dsc[i]; diff --git a/docs/README.config b/docs/README.config index 087bf23f..31095d25 100644 --- a/docs/README.config +++ b/docs/README.config @@ -95,6 +95,9 @@ incoming queue. The following variables have to be set: Accepted, Byhand Done, Holding, New, Reject, Unchecked. An explanation of the function of these directores can be found in README.new-incoming. +Queue::BTSVersionTrack (optional): this directory holds the DebBugs +Version Tracking support files. + ================================================================================ Suite diff --git a/docs/README.first b/docs/README.first index 1f5d23a4..59d796d8 100644 --- a/docs/README.first +++ b/docs/README.first @@ -116,9 +116,11 @@ o If you have an existing archive: * Create the table structure. init_pool.sql contains all SQL statements which are needed for this. After changing all occurences of "projectb" to the name of your database (as defined in DB::Name) you can run: - psql DB::Name < init_pool.sql - * Run alyson: it will populate your database with the values from - katie.conf and apt.conf. - * Copy all templates from the "templates" directory to to the directory - specified in Dir::Templates, and adapt them to your distribution. + psql < init_pool.sql +o Run alyson: it will populate your database with the values from + katie.conf and apt.conf. +o Copy all templates from the "templates" directory to to the directory + specified in Dir::Templates, and adapt them to your distribution. +o Create an 'ftpmaster' group in postgres. +o Run 'psql < add_constraints.sql'. diff --git a/jennifer b/jennifer index 9f98cfb0..c451a053 100755 --- a/jennifer +++ b/jennifer @@ -2,7 +2,7 @@ # Checks Debian packages from Incoming # Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: jennifer,v 1.45 2004-03-11 00:20:51 troup Exp $ +# $Id: jennifer,v 1.46 2004-04-01 17:14:25 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ ################################################################################ -import errno, fcntl, gzip, os, re, shutil, stat, sys, time, traceback; +import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback; import apt_inst, apt_pkg; import db_access, katie, logging, utils; @@ -41,11 +41,13 @@ re_bad_diff = re.compile("^[\-\+][\-\+][\-\+] /dev/null"); re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$"); re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$"); re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$"); +re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)"); +re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.45 $"; +jennifer_version = "$Revision: 1.46 $"; Cnf = None; Options = None; @@ -563,9 +565,13 @@ SELECT c.name FROM binaries b, bin_associations ba, suite s, location l, ############################################################################### -def check_dsc (): +def check_dsc(): global reprocess; + # Ensure there is source to check + if not changes["architecture"].has_key("source"): + return; + # Find the .dsc dsc_filename = None; for file in files.keys(): @@ -576,7 +582,7 @@ def check_dsc (): else: dsc_filename = file; - # If there isn't one, we have nothing to do... + # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: return; @@ -670,17 +676,105 @@ def check_dsc (): ################################################################################ -# dpkg-source broke .diff.gz generation in dpkg 1.8.x; detect the -# resulting bad source packages and reject them. +def get_changelog_versions(source_dir): + """Extracts a the source package and (optionally) grabs the + version history out of debian/changelog for the BTS.""" -def check_diff (): - for filename in files.keys(): - if files[filename]["type"] == "diff.gz": - file = gzip.GzipFile(filename, 'r'); - for line in file.readlines(): - if re_bad_diff.search(line): - reject("%s: invalid .diff.gz produced by a broken version of dpkg-dev 1.8.x." % (filename)); - break; + # Find the .dsc (again) + dsc_filename = None; + for file in files.keys(): + if files[file]["type"] == "dsc": + dsc_filename = file; + + # If there isn't one, we have nothing to do. (We have reject()ed the upload already) + if not dsc_filename: + return; + + # Create a symlink mirror of the source files in our temporary directory + for f in files.keys(): + m = utils.re_issource.match(f); + if m: + src = os.path.join(source_dir, f); + # If a file is missing for whatever reason, give up. + if not os.path.exists(src): + return; + type = m.group(3); + if type == "orig.tar.gz" and pkg.orig_tar_gz: + continue; + else: + dest = os.path.join(os.getcwd(), f); + os.symlink(src, dest); + + # If the orig.tar.gz is not a part of the upload, create a symlink to the + # existing copy. + if pkg.orig_tar_gz: + dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz)); + os.symlink(pkg.orig_tar_gz, dest); + + # Extract the source + cmd = "dpkg-source -sn -x %s" % (dsc_filename); + (result, output) = commands.getstatusoutput(cmd); + if (result != 0): + reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result)); + reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), ""); + return; + + if not Cnf.Find("Dir::Queue::BTSVersionTrack"): + return; + + # Get the upstream version + upstr_version = utils.re_no_epoch.sub('', dsc["version"]); + if re_strip_revision.search(upstr_version): + upstr_version = re_strip_revision.sub('', upstr_version); + + # Ensure the changelog file exists + changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version); + if not os.path.exists(changelog_filename): + reject("%s: debian/changelog not found in extracted source." % (dsc_filename)); + return; + + # Parse the changelog + dsc["bts changelog"] = ""; + changelog_file = utils.open_file(changelog_filename); + for line in changelog_file.readlines(): + m = re_changelog_versions.match(line); + if m: + dsc["bts changelog"] += line; + changelog_file.close(); + + # Check we found at least one revision in the changelog + if not dsc["bts changelog"]: + reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename)); + +######################################## + +def check_source(): + # Bail out if: + # a) there's no source + # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files' + # or c) the orig.tar.gz is MIA + if not changes["architecture"].has_key("source") or reprocess == 2 \ + or pkg.orig_tar_gz == -1: + return; + + # Create a temporary directory to extract the source into + if Options["No-Action"]: + tmpdir = tempfile.mktemp(); + else: + # We're in queue/holding and can create a random directory. + tmpdir = "%s" % (os.getpid()); + os.mkdir(tmpdir); + + # Move into the temporary directory + cwd = os.getcwd(); + os.chdir(tmpdir); + + # Get the changelog version history + get_changelog_versions(cwd); + + # Move back and cleanup the temporary tree + os.chdir(cwd); + shutil.rmtree(tmpdir); ################################################################################ @@ -988,7 +1082,7 @@ def process_it (changes_file): check_distributions(); check_files(); check_dsc(); - check_diff(); + check_source(); check_md5sums(); check_urgency(); check_timestamps(); @@ -1067,7 +1161,7 @@ def main(): if accept_count: sets = "set" if accept_count > 1: - sets = "sets" + sets = "sets"; print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes))); Logger.log(["total",accept_count,accept_bytes]); diff --git a/katie.conf b/katie.conf index 517a5fae..43f7e7dd 100644 --- a/katie.conf +++ b/katie.conf @@ -431,6 +431,7 @@ Dir New "/org/ftp.debian.org/queue/new/"; Reject "/org/ftp.debian.org/queue/reject/"; Unchecked "/org/ftp.debian.org/queue/unchecked/"; + BTSVersionTrack "/org/ftp.debian.org/queue/bts_version_track/"; }; }; diff --git a/katie.py b/katie.py index 776310dd..308806cc 100644 --- a/katie.py +++ b/katie.py @@ -2,7 +2,7 @@ # Utility functions for katie # Copyright (C) 2001, 2002, 2003, 2004 James Troup -# $Id: katie.py,v 1.44 2004-02-27 20:07:40 troup Exp $ +# $Id: katie.py,v 1.45 2004-04-01 17:14:25 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,7 +20,7 @@ ############################################################################### -import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time; +import cPickle, errno, os, pg, re, stat, string, sys, time; import utils, db_access; import apt_inst, apt_pkg; @@ -117,6 +117,7 @@ class Katie: exec "self.pkg.%s.clear();" % (i); self.pkg.orig_tar_id = None; self.pkg.orig_tar_location = ""; + self.pkg.orig_tar_gz = None; ########################################################################### @@ -177,7 +178,8 @@ class Katie: if changes.has_key(i): d_changes[i] = changes[i]; ## dsc - for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]: + for i in [ "source", "version", "maintainer", "fingerprint", "uploaders", + "bts changelog" ]: if dsc.has_key(i): d_dsc[i] = dsc[i]; ## dsc_files @@ -382,14 +384,17 @@ distribution."""; Cnf = self.Cnf; Subst = self.Subst; files = self.pkg.files; + changes = self.pkg.changes; + changes_file = self.pkg.changes_file; + dsc = self.pkg.dsc; print "Accepting." - self.Logger.log(["Accepting changes",self.pkg.changes_file]); + self.Logger.log(["Accepting changes",changes_file]); self.dump_vars(Cnf["Dir::Queue::Accepted"]); # Move all the files into the accepted directory - utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]); + utils.move(changes_file, Cnf["Dir::Queue::Accepted"]); file_keys = files.keys(); for file in file_keys: utils.move(file, Cnf["Dir::Queue::Accepted"]); @@ -405,9 +410,45 @@ distribution."""; utils.send_mail(mail_message) self.announce(short_summary, 1) - # Special support to enable clean auto-building of accepted packages + + ## Helper stuff for DebBugs Version Tracking + if Cnf.Find("Dir::Queue::BTSVersionTrack"): + # ??? once queue/* is cleared on *.d.o and/or reprocessed + # the conditionalization on dsc["bts changelog"] should be + # dropped. + + # Write out the version history from the changelog + if changes["architecture"].has_key("source") and \ + dsc.has_key("bts changelog"): + + temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], + dotprefix=1, perms=0644); + version_history = utils.open_file(temp_filename, 'w'); + version_history.write(dsc["bts changelog"]); + version_history.close(); + filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], + changes_file[:-8]+".versions"); + os.rename(temp_filename, filename); + + # Write out the binary -> source mapping. + temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], + dotprefix=1, perms=0644); + debinfo = utils.open_file(temp_filename, 'w'); + for file in file_keys: + f = files[file]; + if f["type"] == "deb": + line = " ".join([f["package"], f["version"], + f["architecture"], f["source package"], + f["source version"]]); + debinfo.write(line+"\n"); + debinfo.close(); + filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], + changes_file[:-8]+".debinfo"); + os.rename(temp_filename, filename); + + ## Special support to enable clean auto-building of accepted packages self.projectB.query("BEGIN WORK"); - for suite in self.pkg.changes["distribution"].keys(): + for suite in changes["distribution"].keys(): if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"): continue; suite_id = db_access.get_suite_id(suite); @@ -538,9 +579,7 @@ distribution."""; # If we weren't given a manual rejection message, spawn an # editor so the user can add one in... if manual and not reject_message: - temp_filename = tempfile.mktemp(); - fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700); - os.close(fd); + temp_filename = utils.temp_filename(); editor = os.environ.get("EDITOR","vi") answer = 'E'; while answer == 'E': @@ -631,13 +670,13 @@ distribution."""; maps = self.Cnf.ValueList("SuiteMappings")[:] maps.reverse() maps = [ m.split() for m in maps ] - maps = [ (x[1], x[2]) for x in maps + maps = [ (x[1], x[2]) for x in maps if x[0] == "map" or x[0] == "silent-map" ] s = [suite] for x in maps: if x[1] in s and x[0] not in s: s.append(x[0]) - + que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR ")); q = self.projectB.query(que) @@ -797,7 +836,7 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su files = self.pkg.files; dsc_files = self.pkg.dsc_files; legacy_source_untouchable = self.pkg.legacy_source_untouchable; - orig_tar_gz = None; + self.pkg.orig_tar_gz = None; # Try and find all files mentioned in the .dsc. This has # to work harder to cope with the multiple possible @@ -815,7 +854,6 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su # Strip out anything that isn't '%s' or '/%s$' for i in ql: if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file: - self.Logger.log(["check_dsc_against_db",i[2],dsc_file]); ql.remove(i); # "[katie] has not broken them. [katie] has fixed a @@ -848,14 +886,12 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su # Strip out anything that isn't '%s' or '/%s$' for i in ql: if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file: - self.Logger.log(["check_dsc_against_db",i[1],dsc_file]); ql.remove(i); if ql: - # Unfortunately, we make get more than one - # match here if, for example, the package was - # in potato but had a -sa upload in woody. So - # we need to choose the right one. + # Unfortunately, we may get more than one match here if, + # for example, the package was in potato but had an -sa + # upload in woody. So we need to choose the right one. x = ql[0]; # default to something sane in case we don't match any or have only one @@ -877,6 +913,7 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install() # See install() in katie... self.pkg.orig_tar_id = x[3]; + self.pkg.orig_tar_gz = old_file; if suite_type == "legacy" or suite_type == "legacy-mixed": self.pkg.orig_tar_location = "legacy"; else: @@ -895,9 +932,11 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir)); actual_size = os.stat(in_otherdir)[stat.ST_SIZE]; found = in_otherdir; + self.pkg.orig_tar_gz = in_otherdir; if not found: self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file)); + self.pkg.orig_tar_gz = -1; continue; else: self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file)); @@ -907,7 +946,7 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su if actual_size != int(dsc_files[dsc_file]["size"]): self.reject("size for %s doesn't match %s." % (found, file)); - return (self.reject_message, orig_tar_gz); + return (self.reject_message, None); def do_query(self, q): sys.stderr.write("query: \"%s\" ... " % (q));