X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=ceb4a797863b626565febb7a9553293f131232b8;hb=87bf163d12ec8328d87e1b2a2ca2239221a73bd6;hp=813782e23ef38e0dfc7c35818eba74d3f00a37c2;hpb=f8996e240d9d0278bce098e23be63db0bcc6fbee;p=dak.git diff --git a/daklib/queue.py b/daklib/queue.py index 813782e2..ceb4a797 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -1,7 +1,14 @@ #!/usr/bin/env python +# vim:set et sw=4: -# Queue utility functions for dak -# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup +""" +Queue utility functions for dak + +@contact: Debian FTP Master +@copyright: 2001 - 2006 James Troup +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,25 +26,46 @@ ############################################################################### -import cPickle, errno, os, pg, re, stat, sys, time -import apt_inst, apt_pkg -import utils, database +import cPickle +import errno +import os +import pg +import stat +import sys +import time +import apt_inst +import apt_pkg +import utils +import database from dak_exceptions import * +from regexes import re_default_answer, re_fdnic, re_bin_only_nmu from types import * ############################################################################### -re_isanum = re.compile (r"^\d+$") -re_default_answer = re.compile(r"\[(.*)\]") -re_fdnic = re.compile(r"\n\n") -re_bin_only_nmu = re.compile(r"\+b\d+$") - -################################################################################ - # Determine what parts in a .changes are NEW def determine_new(changes, files, projectB, warn=1): + """ + Determine what parts in a C{changes} file are NEW. + + @type changes: Upload.Pkg.changes dict + @param changes: Changes dictionary + + @type files: Upload.Pkg.files dict + @param files: Files dictionary + + @type projectB: pgobject + @param projectB: DB handle + + @type warn: bool + @param warn: Warn if overrides are added for (old)stable + + @rtype: dict + @return: dictionary of NEW components. + + """ new = {} # Build up a list of potentially new things @@ -100,11 +128,21 @@ def determine_new(changes, files, projectB, warn=1): ################################################################################ -def get_type(f): +def get_type(file): + """ + Get the file type of C{file} + + @type file: dict + @param file: file entry + + @rtype: string + @return: filetype + + """ # Determine the type - if f.has_key("dbtype"): - file_type = f["dbtype"] - elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]: + if file.has_key("dbtype"): + file_type = file["dbtype"] + elif file["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]: file_type = "dsc" else: utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type)) @@ -118,9 +156,20 @@ def get_type(f): ################################################################################ -# check if section/priority values are valid + def check_valid(new): + """ + Check if section and priority for NEW packages exist in database. + Additionally does sanity checks: + - debian-installer packages have to be udeb (or source) + - non debian-installer packages can not be udeb + - source priority can only be assigned to dsc file types + + @type new: dict + @param new: Dict of new packages with their section, priority and type. + + """ for pkg in new.keys(): section = new[pkg]["section"] priority = new[pkg]["priority"] @@ -138,9 +187,8 @@ def check_valid(new): ############################################################################### -# Convenience wrapper to carry around all the package information in - class Pkg: + """ Convenience wrapper to carry around all the package information """ def __init__(self, **kwds): self.__dict__.update(kwds) @@ -150,13 +198,21 @@ class Pkg: ############################################################################### class Upload: + """ + Everything that has to do with an upload processed. + """ def __init__(self, Cnf): + """ + Initialize various variables and the global substitution template mappings. + Also connect to the DB and initialize the Database module. + + """ self.Cnf = Cnf self.accept_count = 0 self.accept_bytes = 0L - self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {}, - legacy_source_untouchable = {}) + self.reject_message = "" + self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {}) # Initialize the substitution template mapping global Subst = self.Subst = {} @@ -171,11 +227,11 @@ class Upload: ########################################################################### def init_vars (self): + """ Reset a number of entries from our Pkg object. """ self.pkg.changes.clear() self.pkg.dsc.clear() self.pkg.files.clear() self.pkg.dsc_files.clear() - self.pkg.legacy_source_untouchable.clear() self.pkg.orig_tar_id = None self.pkg.orig_tar_location = "" self.pkg.orig_tar_gz = None @@ -183,6 +239,9 @@ class Upload: ########################################################################### def update_vars (self): + """ + Update our Pkg object by reading a previously created cPickle .dak dumpfile. + """ dump_filename = self.pkg.changes_file[:-8]+".dak" dump_file = utils.open_file(dump_filename) p = cPickle.Unpickler(dump_file) @@ -191,7 +250,6 @@ class Upload: self.pkg.dsc.update(p.load()) self.pkg.files.update(p.load()) self.pkg.dsc_files.update(p.load()) - self.pkg.legacy_source_untouchable.update(p.load()) self.pkg.orig_tar_id = p.load() self.pkg.orig_tar_location = p.load() @@ -200,17 +258,24 @@ class Upload: ########################################################################### - # This could just dump the dictionaries as is, but I'd like to - # avoid this so there's some idea of what process-accepted & - # process-new use from process-unchecked def dump_vars(self, dest_dir): + """ + Dump our Pkg object into a cPickle file. + + @type dest_dir: string + @param dest_dir: Path where the dumpfile should be stored + + @note: This could just dump the dictionaries as is, but I'd like to avoid this so + there's some idea of what process-accepted & process-new use from + process-unchecked. (JT) + + """ changes = self.pkg.changes dsc = self.pkg.dsc files = self.pkg.files dsc_files = self.pkg.dsc_files - legacy_source_untouchable = self.pkg.legacy_source_untouchable orig_tar_id = self.pkg.orig_tar_id orig_tar_location = self.pkg.orig_tar_location @@ -282,7 +347,7 @@ class Upload: d_dsc_files[file_entry][i] = dsc_files[file_entry][i] for i in [ d_changes, d_dsc, d_files, d_dsc_files, - legacy_source_untouchable, orig_tar_id, orig_tar_location ]: + orig_tar_id, orig_tar_location ]: p.dump(i) dump_file.close() @@ -291,6 +356,8 @@ class Upload: # Set up the per-package template substitution mappings def update_subst (self, reject_message = ""): + """ Set up the per-package template substitution mappings """ + Subst = self.Subst changes = self.pkg.changes # If 'dak process-unchecked' crashed out in the right place, architecture may still be a string. @@ -333,6 +400,7 @@ class Upload: ########################################################################### def build_summaries(self): + """ Build a summary of changes the upload introduces. """ changes = self.pkg.changes files = self.pkg.files @@ -343,7 +411,7 @@ class Upload: if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType): changes["distribution"] = {} - override_summary =""; + override_summary ="" file_keys = files.keys() file_keys.sort() for file_entry in file_keys: @@ -389,6 +457,20 @@ class Upload: ########################################################################### def close_bugs (self, summary, action): + """ + Send mail to close bugs as instructed by the closes field in the changes file. + Also add a line to summary if any work was done. + + @type summary: string + @param summary: summary text, as given by L{build_summaries} + + @type action: bool + @param action: Set to false no real action will be done. + + @rtype: string + @return: summary. If action was taken, extended by the list of closed bugs. + + """ changes = self.pkg.changes Subst = self.Subst Cnf = self.Cnf @@ -424,6 +506,19 @@ distribution.""" ########################################################################### def announce (self, short_summary, action): + """ + Send an announce mail about a new upload. + + @type short_summary: string + @param short_summary: Short summary text to include in the mail + + @type action: bool + @param action: Set to false no real action will be done. + + @rtype: string + @return: Textstring about action taken. + + """ Subst = self.Subst Cnf = self.Cnf changes = self.pkg.changes @@ -457,7 +552,24 @@ distribution.""" ########################################################################### - def accept (self, summary, short_summary): + def accept (self, summary, short_summary, targetdir=None): + """ + Accept an upload. + + This moves all files referenced from the .changes into the I{accepted} + queue, sends the accepted mail, announces to lists, closes bugs and + also checks for override disparities. If enabled it will write out + the version history for the BTS Version Tracking and will finally call + L{queue_build}. + + @type summary: string + @param summary: Summary text + + @type short_summary: string + @param short_summary: Short summary + + """ + Cnf = self.Cnf Subst = self.Subst files = self.pkg.files @@ -465,16 +577,19 @@ distribution.""" changes_file = self.pkg.changes_file dsc = self.pkg.dsc + if targetdir is None: + targetdir = Cnf["Dir::Queue::Accepted"] + print "Accepting." self.Logger.log(["Accepting changes",changes_file]) - self.dump_vars(Cnf["Dir::Queue::Accepted"]) + self.dump_vars(targetdir) # Move all the files into the accepted directory - utils.move(changes_file, Cnf["Dir::Queue::Accepted"]) + utils.move(changes_file, targetdir) file_keys = files.keys() for file_entry in file_keys: - utils.move(file_entry, Cnf["Dir::Queue::Accepted"]) + utils.move(file_entry, targetdir) self.accept_bytes += float(files[file_entry]["size"]) self.accept_count += 1 @@ -498,19 +613,18 @@ distribution.""" if changes["architecture"].has_key("source") and \ dsc.has_key("bts changelog"): - temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], - dotprefix=1, perms=0644) - version_history = utils.open_file(temp_filename, 'w') + (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".") + version_history = os.fdopen(fd, 'w') version_history.write(dsc["bts changelog"]) version_history.close() filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], changes_file[:-8]+".versions") os.rename(temp_filename, filename) + os.chmod(filename, 0644) # Write out the binary -> source mapping. - temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], - dotprefix=1, perms=0644) - debinfo = utils.open_file(temp_filename, 'w') + (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".") + debinfo = os.fdopen(fd, 'w') for file_entry in file_keys: f = files[file_entry] if f["type"] == "deb": @@ -522,12 +636,35 @@ distribution.""" filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"], changes_file[:-8]+".debinfo") os.rename(temp_filename, filename) - + os.chmod(filename, 0644) + + # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir! + # we do call queue_build too + # well yes, we'd have had to if we were inserting into accepted + # now. thats database only. + # urgh, that's going to get messy + # so i make the p-n call to it *also* using accepted/ + # but then the packages will be in the queue_build table without the files being there + # as the buildd queue is only regenerated whenever unchecked runs + # ah, good point + # so it will work out, as unchecked move it over + # that's all completely sick + # yes self.queue_build("accepted", Cnf["Dir::Queue::Accepted"]) ########################################################################### def queue_build (self, queue, path): + """ + Prepare queue_build database table used for incoming autobuild support. + + @type queue: string + @param queue: queue name + + @type path: string + @param path: path for the queue file entries/link destinations + """ + Cnf = self.Cnf Subst = self.Subst files = self.pkg.files @@ -586,6 +723,16 @@ distribution.""" ########################################################################### def check_override (self): + """ + Checks override entries for validity. Mails "Override disparity" warnings, + if that feature is enabled. + + Abandons the check if + - this is a non-sourceful upload + - override disparity checks are disabled + - mail sending is disabled + + """ Subst = self.Subst changes = self.pkg.changes files = self.pkg.files @@ -624,10 +771,16 @@ distribution.""" ########################################################################### def force_reject (self, files): - """Forcefully move files from the current directory to the - reject directory. If any file already exists in the reject - directory it will be moved to the morgue to make way for - the new file.""" + """ + Forcefully move files from the current directory to the + reject directory. If any file already exists in the reject + directory it will be moved to the morgue to make way for + the new file. + + @type files: dict + @param files: file dictionary + + """ Cnf = self.Cnf @@ -665,11 +818,29 @@ distribution.""" ########################################################################### - def do_reject (self, manual = 0, reject_message = ""): + def do_reject (self, manual = 0, reject_message = "", note = ""): + """ + Reject an upload. If called without a reject message or C{manual} is + true, spawn an editor so the user can write one. + + @type manual: bool + @param manual: manual or automated rejection + + @type reject_message: string + @param reject_message: A reject message + + @return: 0 + + """ # If we weren't given a manual rejection message, spawn an # editor so the user can add one in... if manual and not reject_message: - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() + temp_file = os.fdopen(fd, 'w') + if len(note) > 0: + for line in note: + temp_file.write(line) + temp_file.close() editor = os.environ.get("EDITOR","vi") answer = 'E' while answer == 'E': @@ -740,13 +911,26 @@ distribution.""" ################################################################################ - # Ensure that source exists somewhere in the archive for the binary - # upload being processed. - # - # (1) exact match => 1.0-3 - # (2) Bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 - def source_exists (self, package, source_version, suites = ["any"]): + """ + Ensure that source exists somewhere in the archive for the binary + upload being processed. + 1. exact match => 1.0-3 + 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1 + + @type package: string + @param package: package source name + + @type source_version: string + @param source_version: expected source version + + @type suites: list + @param suites: list of suites to check in, default I{any} + + @rtype: int + @return: returns 1 if a source with expected version is found, otherwise 0 + + """ okay = 1 for suite in suites: if suite == "any": @@ -789,6 +973,27 @@ distribution.""" ################################################################################ def in_override_p (self, package, component, suite, binary_type, file): + """ + Check if a package already has override entries in the DB + + @type package: string + @param package: package name + + @type component: string + @param component: database id of the component, as returned by L{database.get_component_id} + + @type suite: int + @param suite: database id of the suite, as returned by L{database.get_suite_id} + + @type binary_type: string + @param binary_type: type of the package + + @type file: string + @param file: filename we check + + @return: the database result. But noone cares anyway. + + """ files = self.pkg.files if binary_type == "": # must be source @@ -828,6 +1033,16 @@ distribution.""" ################################################################################ def reject (self, str, prefix="Rejected: "): + """ + Add C{str} to reject_message. Adds C{prefix}, by default "Rejected: " + + @type str: string + @param str: Reject text + + @type prefix: string + @param prefix: Prefix text, default Rejected: + + """ if str: # Unlike other rejects we add new lines first to avoid trailing # new lines when this message is passed back up to a caller. @@ -838,6 +1053,7 @@ distribution.""" ################################################################################ def get_anyversion(self, query_result, suite): + """ """ anyversion=None anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite)) for (v, s) in query_result: @@ -848,22 +1064,26 @@ distribution.""" ################################################################################ - def cross_suite_version_check(self, query_result, file, new_version): - """Ensure versions are newer than existing packages in target + def cross_suite_version_check(self, query_result, file, new_version, + sourceful=False): + """ + Ensure versions are newer than existing packages in target suites and that cross-suite version checking rules as - set out in the conf file are satisfied.""" + set out in the conf file are satisfied. + + """ # Check versions for each target suite for target_suite in self.pkg.changes["distribution"].keys(): - must_be_newer_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ] - must_be_older_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ] + must_be_newer_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ] + must_be_older_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ] # Enforce "must be newer than target suite" even if conffile omits it if target_suite not in must_be_newer_than: must_be_newer_than.append(target_suite) for entry in query_result: existent_version = entry[0] suite = entry[1] - if suite in must_be_newer_than and \ + if suite in must_be_newer_than and sourceful and \ apt_pkg.VersionCompare(new_version, existent_version) < 1: self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)) if suite in must_be_older_than and \ @@ -914,6 +1134,9 @@ distribution.""" ################################################################################ def check_binary_against_db(self, file): + """ + + """ self.reject_message = "" files = self.pkg.files @@ -925,7 +1148,8 @@ SELECT b.version, su.suite_name FROM binaries b, bin_associations ba, suite su, AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id""" % (files[file]["package"], files[file]["architecture"])) - self.cross_suite_version_check(q.getresult(), file, files[file]["version"]) + self.cross_suite_version_check(q.getresult(), file, + files[file]["version"], sourceful=False) # Check for any existing copies of the file q = self.projectB.query(""" @@ -943,6 +1167,8 @@ SELECT b.id FROM binaries b, architecture a ################################################################################ def check_source_against_db(self, file): + """ + """ self.reject_message = "" dsc = self.pkg.dsc @@ -950,25 +1176,27 @@ SELECT b.id FROM binaries b, architecture a q = self.projectB.query(""" SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source"))) - self.cross_suite_version_check(q.getresult(), file, dsc.get("version")) + self.cross_suite_version_check(q.getresult(), file, dsc.get("version"), + sourceful=True) return self.reject_message ################################################################################ - # **WARNING** - # NB: this function can remove entries from the 'files' index [if - # the .orig.tar.gz is a duplicate of the one in the archive]; if - # you're iterating over 'files' and call this function as part of - # the loop, be sure to add a check to the top of the loop to - # ensure you haven't just tried to dereference the deleted entry. - # **WARNING** def check_dsc_against_db(self, file): + """ + + @warning: NB: this function can remove entries from the 'files' index [if + the .orig.tar.gz is a duplicate of the one in the archive]; if + you're iterating over 'files' and call this function as part of + the loop, be sure to add a check to the top of the loop to + ensure you haven't just tried to dereference the deleted entry. + + """ self.reject_message = "" files = self.pkg.files dsc_files = self.pkg.dsc_files - legacy_source_untouchable = self.pkg.legacy_source_untouchable self.pkg.orig_tar_gz = None # Try and find all files mentioned in the .dsc. This has @@ -1028,7 +1256,8 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su # for example, the package was in potato but had an -sa # upload in woody. So we need to choose the right one. - x = ql[0]; # default to something sane in case we don't match any or have only one + # default to something sane in case we don't match any or have only one + x = ql[0] if len(ql) > 1: for i in ql: @@ -1039,8 +1268,6 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su actual_size = os.stat(old_file)[stat.ST_SIZE] if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]): x = i - else: - legacy_source_untouchable[i[3]] = "" old_file = x[0] + x[1] old_file_fh = utils.open_file(old_file) @@ -1049,14 +1276,12 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su actual_size = os.stat(old_file)[stat.ST_SIZE] found = old_file suite_type = x[2] - dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install() + # need this for updating dsc_files in install() + dsc_files[dsc_file]["files id"] = x[3] # See install() in process-accepted... self.pkg.orig_tar_id = x[3] self.pkg.orig_tar_gz = old_file - if suite_type == "legacy" or suite_type == "legacy-mixed": - self.pkg.orig_tar_location = "legacy" - else: - self.pkg.orig_tar_location = x[4] + self.pkg.orig_tar_location = x[4] else: # Not there? Check the queue directories... @@ -1091,10 +1316,20 @@ SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su return (self.reject_message, None) - def do_query(self, q): - sys.stderr.write("query: \"%s\" ... " % (q)) + def do_query(self, query): + """ + Executes a database query. Writes statistics / timing to stderr. + + @type query: string + @param query: database query string, passed unmodified + + @return: db result + + @warning: The query is passed B{unmodified}, so be careful what you use this for. + """ + sys.stderr.write("query: \"%s\" ... " % (query)) before = time.time() - r = self.projectB.query(q) + r = self.projectB.query(query) time_diff = time.time()-before sys.stderr.write("took %.3f seconds.\n" % (time_diff)) return r