#!/usr/bin/env python
+# vim:set et sw=4:
-# Queue utility functions for dak
-# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
+"""
+Queue utility functions for dak
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2001 - 2006 James Troup <james@nocrew.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
###############################################################################
-import cPickle, errno, os, pg, re, stat, sys, time
-import apt_inst, apt_pkg
-import utils, database
+import cPickle
+import errno
+import os
+import pg
+import stat
+import sys
+import time
+import apt_inst
+import apt_pkg
+import utils
+import database
from dak_exceptions import *
+from regexes import re_default_answer, re_fdnic, re_bin_only_nmu
from types import *
###############################################################################
-re_isanum = re.compile (r"^\d+$")
-re_default_answer = re.compile(r"\[(.*)\]")
-re_fdnic = re.compile(r"\n\n")
-re_bin_only_nmu = re.compile(r"\+b\d+$")
-
-################################################################################
-
# Determine what parts in a .changes are NEW
def determine_new(changes, files, projectB, warn=1):
+ """
+ Determine what parts in a C{changes} file are NEW.
+
+ @type changes: Upload.Pkg.changes dict
+ @param changes: Changes dictionary
+
+ @type files: Upload.Pkg.files dict
+ @param files: Files dictionary
+
+ @type projectB: pgobject
+ @param projectB: DB handle
+
+ @type warn: bool
+ @param warn: Warn if overrides are added for (old)stable
+
+ @rtype: dict
+ @return: dictionary of NEW components.
+
+ """
new = {}
# Build up a list of potentially new things
################################################################################
-def get_type(f):
+def get_type(file):
+ """
+ Get the file type of C{file}
+
+ @type file: dict
+ @param file: file entry
+
+ @rtype: string
+ @return: filetype
+
+ """
# Determine the type
- if f.has_key("dbtype"):
- file_type = f["dbtype"]
- elif f["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
+ if file.has_key("dbtype"):
+ file_type = file["dbtype"]
+ elif file["type"] in [ "orig.tar.gz", "orig.tar.bz2", "tar.gz", "tar.bz2", "diff.gz", "diff.bz2", "dsc" ]:
file_type = "dsc"
else:
utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (file_type))
################################################################################
-# check if section/priority values are valid
+
def check_valid(new):
+ """
+ Check if section and priority for NEW packages exist in database.
+ Additionally does sanity checks:
+ - debian-installer packages have to be udeb (or source)
+ - non debian-installer packages can not be udeb
+ - source priority can only be assigned to dsc file types
+
+ @type new: dict
+ @param new: Dict of new packages with their section, priority and type.
+
+ """
for pkg in new.keys():
section = new[pkg]["section"]
priority = new[pkg]["priority"]
###############################################################################
-# Convenience wrapper to carry around all the package information in
-
class Pkg:
+ """ Convenience wrapper to carry around all the package information """
def __init__(self, **kwds):
self.__dict__.update(kwds)
###############################################################################
class Upload:
+ """
+ Everything that has to do with an upload processed.
+ """
def __init__(self, Cnf):
+ """
+ Initialize various variables and the global substitution template mappings.
+ Also connect to the DB and initialize the Database module.
+
+ """
self.Cnf = Cnf
self.accept_count = 0
self.accept_bytes = 0L
- self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {},
- legacy_source_untouchable = {})
+ self.reject_message = ""
+ self.pkg = Pkg(changes = {}, dsc = {}, dsc_files = {}, files = {})
# Initialize the substitution template mapping global
Subst = self.Subst = {}
###########################################################################
def init_vars (self):
+ """ Reset a number of entries from our Pkg object. """
self.pkg.changes.clear()
self.pkg.dsc.clear()
self.pkg.files.clear()
self.pkg.dsc_files.clear()
- self.pkg.legacy_source_untouchable.clear()
self.pkg.orig_tar_id = None
self.pkg.orig_tar_location = ""
self.pkg.orig_tar_gz = None
###########################################################################
def update_vars (self):
+ """
+ Update our Pkg object by reading a previously created cPickle .dak dumpfile.
+ """
dump_filename = self.pkg.changes_file[:-8]+".dak"
dump_file = utils.open_file(dump_filename)
p = cPickle.Unpickler(dump_file)
self.pkg.dsc.update(p.load())
self.pkg.files.update(p.load())
self.pkg.dsc_files.update(p.load())
- self.pkg.legacy_source_untouchable.update(p.load())
self.pkg.orig_tar_id = p.load()
self.pkg.orig_tar_location = p.load()
###########################################################################
- # This could just dump the dictionaries as is, but I'd like to
- # avoid this so there's some idea of what process-accepted &
- # process-new use from process-unchecked
def dump_vars(self, dest_dir):
+ """
+ Dump our Pkg object into a cPickle file.
+
+ @type dest_dir: string
+ @param dest_dir: Path where the dumpfile should be stored
+
+ @note: This could just dump the dictionaries as is, but I'd like to avoid this so
+ there's some idea of what process-accepted & process-new use from
+ process-unchecked. (JT)
+
+ """
changes = self.pkg.changes
dsc = self.pkg.dsc
files = self.pkg.files
dsc_files = self.pkg.dsc_files
- legacy_source_untouchable = self.pkg.legacy_source_untouchable
orig_tar_id = self.pkg.orig_tar_id
orig_tar_location = self.pkg.orig_tar_location
d_dsc_files[file_entry][i] = dsc_files[file_entry][i]
for i in [ d_changes, d_dsc, d_files, d_dsc_files,
- legacy_source_untouchable, orig_tar_id, orig_tar_location ]:
+ orig_tar_id, orig_tar_location ]:
p.dump(i)
dump_file.close()
# Set up the per-package template substitution mappings
def update_subst (self, reject_message = ""):
+ """ Set up the per-package template substitution mappings """
+
Subst = self.Subst
changes = self.pkg.changes
# If 'dak process-unchecked' crashed out in the right place, architecture may still be a string.
###########################################################################
def build_summaries(self):
+ """ Build a summary of changes the upload introduces. """
changes = self.pkg.changes
files = self.pkg.files
if not changes.has_key("distribution") or not isinstance(changes["distribution"], DictType):
changes["distribution"] = {}
- override_summary ="";
+ override_summary =""
file_keys = files.keys()
file_keys.sort()
for file_entry in file_keys:
###########################################################################
def close_bugs (self, summary, action):
+ """
+ Send mail to close bugs as instructed by the closes field in the changes file.
+ Also add a line to summary if any work was done.
+
+ @type summary: string
+ @param summary: summary text, as given by L{build_summaries}
+
+ @type action: bool
+ @param action: Set to false no real action will be done.
+
+ @rtype: string
+ @return: summary. If action was taken, extended by the list of closed bugs.
+
+ """
changes = self.pkg.changes
Subst = self.Subst
Cnf = self.Cnf
###########################################################################
def announce (self, short_summary, action):
+ """
+ Send an announce mail about a new upload.
+
+ @type short_summary: string
+ @param short_summary: Short summary text to include in the mail
+
+ @type action: bool
+ @param action: Set to false no real action will be done.
+
+ @rtype: string
+ @return: Textstring about action taken.
+
+ """
Subst = self.Subst
Cnf = self.Cnf
changes = self.pkg.changes
###########################################################################
- def accept (self, summary, short_summary):
+ def accept (self, summary, short_summary, targetdir=None):
+ """
+ Accept an upload.
+
+ This moves all files referenced from the .changes into the I{accepted}
+ queue, sends the accepted mail, announces to lists, closes bugs and
+ also checks for override disparities. If enabled it will write out
+ the version history for the BTS Version Tracking and will finally call
+ L{queue_build}.
+
+ @type summary: string
+ @param summary: Summary text
+
+ @type short_summary: string
+ @param short_summary: Short summary
+
+ """
+
Cnf = self.Cnf
Subst = self.Subst
files = self.pkg.files
changes_file = self.pkg.changes_file
dsc = self.pkg.dsc
+ if targetdir is None:
+ targetdir = Cnf["Dir::Queue::Accepted"]
+
print "Accepting."
self.Logger.log(["Accepting changes",changes_file])
- self.dump_vars(Cnf["Dir::Queue::Accepted"])
+ self.dump_vars(targetdir)
# Move all the files into the accepted directory
- utils.move(changes_file, Cnf["Dir::Queue::Accepted"])
+ utils.move(changes_file, targetdir)
file_keys = files.keys()
for file_entry in file_keys:
- utils.move(file_entry, Cnf["Dir::Queue::Accepted"])
+ utils.move(file_entry, targetdir)
self.accept_bytes += float(files[file_entry]["size"])
self.accept_count += 1
if changes["architecture"].has_key("source") and \
dsc.has_key("bts changelog"):
- temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
- dotprefix=1, perms=0644)
- version_history = utils.open_file(temp_filename, 'w')
+ (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ version_history = os.fdopen(fd, 'w')
version_history.write(dsc["bts changelog"])
version_history.close()
filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
changes_file[:-8]+".versions")
os.rename(temp_filename, filename)
+ os.chmod(filename, 0644)
# Write out the binary -> source mapping.
- temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
- dotprefix=1, perms=0644)
- debinfo = utils.open_file(temp_filename, 'w')
+ (fd, temp_filename) = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+ debinfo = os.fdopen(fd, 'w')
for file_entry in file_keys:
f = files[file_entry]
if f["type"] == "deb":
filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
changes_file[:-8]+".debinfo")
os.rename(temp_filename, filename)
-
+ os.chmod(filename, 0644)
+
+ # Its is Cnf["Dir::Queue::Accepted"] here, not targetdir!
+ # <Ganneff> we do call queue_build too
+ # <mhy> well yes, we'd have had to if we were inserting into accepted
+ # <Ganneff> now. thats database only.
+ # <mhy> urgh, that's going to get messy
+ # <Ganneff> so i make the p-n call to it *also* using accepted/
+ # <mhy> but then the packages will be in the queue_build table without the files being there
+ # <Ganneff> as the buildd queue is only regenerated whenever unchecked runs
+ # <mhy> ah, good point
+ # <Ganneff> so it will work out, as unchecked move it over
+ # <mhy> that's all completely sick
+ # <Ganneff> yes
self.queue_build("accepted", Cnf["Dir::Queue::Accepted"])
###########################################################################
def queue_build (self, queue, path):
+ """
+ Prepare queue_build database table used for incoming autobuild support.
+
+ @type queue: string
+ @param queue: queue name
+
+ @type path: string
+ @param path: path for the queue file entries/link destinations
+ """
+
Cnf = self.Cnf
Subst = self.Subst
files = self.pkg.files
###########################################################################
def check_override (self):
+ """
+ Checks override entries for validity. Mails "Override disparity" warnings,
+ if that feature is enabled.
+
+ Abandons the check if
+ - this is a non-sourceful upload
+ - override disparity checks are disabled
+ - mail sending is disabled
+
+ """
Subst = self.Subst
changes = self.pkg.changes
files = self.pkg.files
###########################################################################
def force_reject (self, files):
- """Forcefully move files from the current directory to the
- reject directory. If any file already exists in the reject
- directory it will be moved to the morgue to make way for
- the new file."""
+ """
+ Forcefully move files from the current directory to the
+ reject directory. If any file already exists in the reject
+ directory it will be moved to the morgue to make way for
+ the new file.
+
+ @type files: dict
+ @param files: file dictionary
+
+ """
Cnf = self.Cnf
###########################################################################
- def do_reject (self, manual = 0, reject_message = ""):
+ def do_reject (self, manual = 0, reject_message = "", note = ""):
+ """
+ Reject an upload. If called without a reject message or C{manual} is
+ true, spawn an editor so the user can write one.
+
+ @type manual: bool
+ @param manual: manual or automated rejection
+
+ @type reject_message: string
+ @param reject_message: A reject message
+
+ @return: 0
+
+ """
# If we weren't given a manual rejection message, spawn an
# editor so the user can add one in...
if manual and not reject_message:
- temp_filename = utils.temp_filename()
+ (fd, temp_filename) = utils.temp_filename()
+ temp_file = os.fdopen(fd, 'w')
+ if len(note) > 0:
+ for line in note:
+ temp_file.write(line)
+ temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
while answer == 'E':
################################################################################
- # Ensure that source exists somewhere in the archive for the binary
- # upload being processed.
- #
- # (1) exact match => 1.0-3
- # (2) Bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
-
def source_exists (self, package, source_version, suites = ["any"]):
+ """
+ Ensure that source exists somewhere in the archive for the binary
+ upload being processed.
+ 1. exact match => 1.0-3
+ 2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
+
+ @type package: string
+ @param package: package source name
+
+ @type source_version: string
+ @param source_version: expected source version
+
+ @type suites: list
+ @param suites: list of suites to check in, default I{any}
+
+ @rtype: int
+ @return: returns 1 if a source with expected version is found, otherwise 0
+
+ """
okay = 1
for suite in suites:
if suite == "any":
################################################################################
def in_override_p (self, package, component, suite, binary_type, file):
+ """
+ Check if a package already has override entries in the DB
+
+ @type package: string
+ @param package: package name
+
+ @type component: string
+ @param component: database id of the component, as returned by L{database.get_component_id}
+
+ @type suite: int
+ @param suite: database id of the suite, as returned by L{database.get_suite_id}
+
+ @type binary_type: string
+ @param binary_type: type of the package
+
+ @type file: string
+ @param file: filename we check
+
+ @return: the database result. But noone cares anyway.
+
+ """
files = self.pkg.files
if binary_type == "": # must be source
################################################################################
def reject (self, str, prefix="Rejected: "):
+ """
+ Add C{str} to reject_message. Adds C{prefix}, by default "Rejected: "
+
+ @type str: string
+ @param str: Reject text
+
+ @type prefix: string
+ @param prefix: Prefix text, default Rejected:
+
+ """
if str:
# Unlike other rejects we add new lines first to avoid trailing
# new lines when this message is passed back up to a caller.
################################################################################
def get_anyversion(self, query_result, suite):
+ """ """
anyversion=None
anysuite = [suite] + self.Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
for (v, s) in query_result:
################################################################################
- def cross_suite_version_check(self, query_result, file, new_version):
- """Ensure versions are newer than existing packages in target
+ def cross_suite_version_check(self, query_result, file, new_version,
+ sourceful=False):
+ """
+ Ensure versions are newer than existing packages in target
suites and that cross-suite version checking rules as
- set out in the conf file are satisfied."""
+ set out in the conf file are satisfied.
+
+ """
# Check versions for each target suite
for target_suite in self.pkg.changes["distribution"].keys():
- must_be_newer_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
- must_be_older_than = [ i.lower for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+ must_be_newer_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
+ must_be_older_than = [ i.lower() for i in self.Cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
# Enforce "must be newer than target suite" even if conffile omits it
if target_suite not in must_be_newer_than:
must_be_newer_than.append(target_suite)
for entry in query_result:
existent_version = entry[0]
suite = entry[1]
- if suite in must_be_newer_than and \
+ if suite in must_be_newer_than and sourceful and \
apt_pkg.VersionCompare(new_version, existent_version) < 1:
self.reject("%s: old version (%s) in %s >= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite))
if suite in must_be_older_than and \
################################################################################
def check_binary_against_db(self, file):
+ """
+
+ """
self.reject_message = ""
files = self.pkg.files
AND ba.bin = b.id AND ba.suite = su.id AND b.architecture = a.id"""
% (files[file]["package"],
files[file]["architecture"]))
- self.cross_suite_version_check(q.getresult(), file, files[file]["version"])
+ self.cross_suite_version_check(q.getresult(), file,
+ files[file]["version"], sourceful=False)
# Check for any existing copies of the file
q = self.projectB.query("""
################################################################################
def check_source_against_db(self, file):
+ """
+ """
self.reject_message = ""
dsc = self.pkg.dsc
q = self.projectB.query("""
SELECT s.version, su.suite_name FROM source s, src_associations sa, suite su
WHERE s.source = '%s' AND sa.source = s.id AND sa.suite = su.id""" % (dsc.get("source")))
- self.cross_suite_version_check(q.getresult(), file, dsc.get("version"))
+ self.cross_suite_version_check(q.getresult(), file, dsc.get("version"),
+ sourceful=True)
return self.reject_message
################################################################################
- # **WARNING**
- # NB: this function can remove entries from the 'files' index [if
- # the .orig.tar.gz is a duplicate of the one in the archive]; if
- # you're iterating over 'files' and call this function as part of
- # the loop, be sure to add a check to the top of the loop to
- # ensure you haven't just tried to dereference the deleted entry.
- # **WARNING**
def check_dsc_against_db(self, file):
+ """
+
+ @warning: NB: this function can remove entries from the 'files' index [if
+ the .orig.tar.gz is a duplicate of the one in the archive]; if
+ you're iterating over 'files' and call this function as part of
+ the loop, be sure to add a check to the top of the loop to
+ ensure you haven't just tried to dereference the deleted entry.
+
+ """
self.reject_message = ""
files = self.pkg.files
dsc_files = self.pkg.dsc_files
- legacy_source_untouchable = self.pkg.legacy_source_untouchable
self.pkg.orig_tar_gz = None
# Try and find all files mentioned in the .dsc. This has
# for example, the package was in potato but had an -sa
# upload in woody. So we need to choose the right one.
- x = ql[0]; # default to something sane in case we don't match any or have only one
+ # default to something sane in case we don't match any or have only one
+ x = ql[0]
if len(ql) > 1:
for i in ql:
actual_size = os.stat(old_file)[stat.ST_SIZE]
if actual_md5 == dsc_files[dsc_file]["md5sum"] and actual_size == int(dsc_files[dsc_file]["size"]):
x = i
- else:
- legacy_source_untouchable[i[3]] = ""
old_file = x[0] + x[1]
old_file_fh = utils.open_file(old_file)
actual_size = os.stat(old_file)[stat.ST_SIZE]
found = old_file
suite_type = x[2]
- dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
+ # need this for updating dsc_files in install()
+ dsc_files[dsc_file]["files id"] = x[3]
# See install() in process-accepted...
self.pkg.orig_tar_id = x[3]
self.pkg.orig_tar_gz = old_file
- if suite_type == "legacy" or suite_type == "legacy-mixed":
- self.pkg.orig_tar_location = "legacy"
- else:
- self.pkg.orig_tar_location = x[4]
+ self.pkg.orig_tar_location = x[4]
else:
# Not there? Check the queue directories...
return (self.reject_message, None)
- def do_query(self, q):
- sys.stderr.write("query: \"%s\" ... " % (q))
+ def do_query(self, query):
+ """
+ Executes a database query. Writes statistics / timing to stderr.
+
+ @type query: string
+ @param query: database query string, passed unmodified
+
+ @return: db result
+
+ @warning: The query is passed B{unmodified}, so be careful what you use this for.
+ """
+ sys.stderr.write("query: \"%s\" ... " % (query))
before = time.time()
- r = self.projectB.query(q)
+ r = self.projectB.query(query)
time_diff = time.time()-before
sys.stderr.write("took %.3f seconds.\n" % (time_diff))
return r
projects / dak.git / blobdiff