Merge commit 'ftpmaster/master' into sqlalchemy

[dak.git] / daklib / utils.py

diff --git a/daklib/utils.py b/daklib/utils.py
index 86e51a2f5c852f5b8c53b42741db14fcb96265ef..5edf2cb1eeddcd34a3dd7f5237b029a3f863ba8c 100755 (executable)
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 # vim:set et ts=4 sw=4:
 
 #!/usr/bin/env python
 # vim:set et ts=4 sw=4:
 

-# Utility functions
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
+"""Utility functions

 
 

-################################################################################
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
+@license: GNU General Public License version 2 or later
+"""

 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by


@@ -20,49 +22,68 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 

-################################################################################
-
-import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \
-       sys, tempfile, traceback, stat
+import codecs
+import commands
+import email.Header
+import os
+import pwd
+import select
+import socket
+import shutil
+import sys
+import tempfile
+import traceback
+import stat

 import apt_pkg
 import apt_pkg

-import database
+import time
+import re
+import string
+import email as modemail
+
+from dbconn import DBConn, get_architecture, get_component, get_suite

 from dak_exceptions import *
 from dak_exceptions import *

+from textutils import fix_maintainer
+from regexes import re_html_escaping, html_escaping, re_single_line_field, \
+                    re_multi_line_field, re_srchasver, re_verwithext, \
+                    re_parse_maintainer, re_taint_free, re_gpg_uid, re_re_mark, \
+                    re_whitespace_comment

 
 ################################################################################
 
 
 ################################################################################
 

-re_comments = re.compile(r"\#.*")
-re_no_epoch = re.compile(r"^\d+\:")
-re_no_revision = re.compile(r"-[^-]+$")
-re_arch_from_filename = re.compile(r"/binary-[^/]+/")
-re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)")
-re_isadeb = re.compile (r"(.+?)_(.+?)_(.+)\.u?deb$")
-re_issource = re.compile (r"(.+)_(.+?)\.(orig\.tar\.gz|diff\.gz|tar\.gz|dsc)$")
+default_config = "/etc/dak/dak.conf"     #: default dak config, defines host properties
+default_apt_config = "/etc/dak/apt.conf" #: default apt config, not normally used

 
 

-re_single_line_field = re.compile(r"^(\S*)\s*:\s*(.*)")
-re_multi_line_field = re.compile(r"^\s(.*)")
-re_taint_free = re.compile(r"^[-+~/\.\w]+$")
+alias_cache = None        #: Cache for email alias checks
+key_uid_email_cache = {}  #: Cache for email addresses from gpg key uids

 
 

-re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")
-re_gpg_uid = re.compile('^uid.*<([^>]*)>')
+# (hashname, function, earliest_changes_version)
+known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
+                ("sha256", apt_pkg.sha256sum, (1, 8))] #: hashes we accept for entries in .changes/.dsc

 
 

-re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$")
-re_verwithext = re.compile(r"^(\d+)(?:\.(\d+))(?:\s+\((\S+)\))?$")
+################################################################################

 
 

-re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$")
+def html_escape(s):
+    """ Escape html chars """
+    return re_html_escaping.sub(lambda x: html_escaping.get(x.group(0)), s)

 
 

-default_config = "/etc/dak/dak.conf"
-default_apt_config = "/etc/dak/apt.conf"
+################################################################################

 
 

-alias_cache = None
-key_uid_email_cache = {}
+def open_file(filename, mode='r'):
+    """
+    Open C{file}, return fileobject.

 
 

-# (hashname, function, earliest_changes_version)
-known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
-                ("sha256", apt_pkg.sha256sum, (1, 8))]
+    @type filename: string
+    @param filename: path/filename to open

 
 

-################################################################################
+    @type mode: string
+    @param mode: open mode

 
 

-def open_file(filename, mode='r'):
+    @rtype: fileobject
+    @return: open fileobject
+
+    @raise CantOpenError: If IOError is raised by open, reraise it as CantOpenError.
+
+    """

     try:
         f = open(filename, mode)
     except IOError:
     try:
         f = open(filename, mode)
     except IOError:


@@ -192,29 +213,34 @@ def parse_deb822(contents, signing_rules=0):
 ################################################################################
 
 def parse_changes(filename, signing_rules=0):
 ################################################################################
 
 def parse_changes(filename, signing_rules=0):

-    """Parses a changes file and returns a dictionary where each field is a
-key.  The mandatory first argument is the filename of the .changes
-file.
+    """
+    Parses a changes file and returns a dictionary where each field is a
+    key.  The mandatory first argument is the filename of the .changes
+    file.

 
 

-signing_rules is an optional argument:
+    signing_rules is an optional argument:

 
 

- o If signing_rules == -1, no signature is required.
- o If signing_rules == 0 (the default), a signature is required.
- o If signing_rules == 1, it turns on the same strict format checking
-   as dpkg-source.
+      - If signing_rules == -1, no signature is required.
+      - If signing_rules == 0 (the default), a signature is required.
+      - If signing_rules == 1, it turns on the same strict format checking
+        as dpkg-source.

 
 

-The rules for (signing_rules == 1)-mode are:
+    The rules for (signing_rules == 1)-mode are:

 
 

-  o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
-    followed by any PGP header data and must end with a blank line.
+      - The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----"
+        followed by any PGP header data and must end with a blank line.

 
 

-  o The data section must end with a blank line and must be followed by
-    "-----BEGIN PGP SIGNATURE-----".
-"""
+      - The data section must end with a blank line and must be followed by
+        "-----BEGIN PGP SIGNATURE-----".
+    """

 
     changes_in = open_file(filename)
     content = changes_in.read()
     changes_in.close()
 
     changes_in = open_file(filename)
     content = changes_in.read()
     changes_in.close()

+    try:
+        unicode(content, 'utf-8')
+    except UnicodeError:
+        raise ChangesUnicodeError, "Changes file not proper utf-8"

     return parse_deb822(content, signing_rules)
 
 ################################################################################
     return parse_deb822(content, signing_rules)
 
 ################################################################################


@@ -225,9 +251,11 @@ def hash_key(hashname):
 ################################################################################
 
 def create_hash(where, files, hashname, hashfunc):
 ################################################################################
 
 def create_hash(where, files, hashname, hashfunc):

-    """create_hash extends the passed files dict with the given hash by
+    """
+    create_hash extends the passed files dict with the given hash by

     iterating over all files on disk and passing them to the hashing
     iterating over all files on disk and passing them to the hashing

-    function given."""
+    function given.
+    """

 
     rejmsg = []
     for f in files.keys():
 
     rejmsg = []
     for f in files.keys():


@@ -235,6 +263,7 @@ def create_hash(where, files, hashname, hashfunc):
             file_handle = open_file(f)
         except CantOpenError:
             rejmsg.append("Could not open file %s for checksumming" % (f))
             file_handle = open_file(f)
         except CantOpenError:
             rejmsg.append("Could not open file %s for checksumming" % (f))

+            continue

 
         files[f][hash_key(hashname)] = hashfunc(file_handle)
 
 
         files[f][hash_key(hashname)] = hashfunc(file_handle)
 


@@ -244,9 +273,11 @@ def create_hash(where, files, hashname, hashfunc):
 ################################################################################
 
 def check_hash(where, files, hashname, hashfunc):
 ################################################################################
 
 def check_hash(where, files, hashname, hashfunc):

-    """check_hash checks the given hash in the files dict against the actual
+    """
+    check_hash checks the given hash in the files dict against the actual

     files on disk.  The hash values need to be present consistently in
     files on disk.  The hash values need to be present consistently in

-    all file entries.  It does not modify its input in any way."""
+    all file entries.  It does not modify its input in any way.
+    """

 
     rejmsg = []
     for f in files.keys():
 
     rejmsg = []
     for f in files.keys():


@@ -277,8 +308,10 @@ def check_hash(where, files, hashname, hashfunc):
 ################################################################################
 
 def check_size(where, files):
 ################################################################################
 
 def check_size(where, files):

-    """check_size checks the file sizes in the passed files dict against the
-    files on disk."""
+    """
+    check_size checks the file sizes in the passed files dict against the
+    files on disk.
+    """

 
     rejmsg = []
     for f in files.keys():
 
     rejmsg = []
     for f in files.keys():


@@ -300,8 +333,10 @@ def check_size(where, files):
 ################################################################################
 
 def check_hash_fields(what, manifest):
 ################################################################################
 
 def check_hash_fields(what, manifest):

-    """check_hash_fields ensures that there are no checksum fields in the
-    given dict that we do not know about."""
+    """
+    check_hash_fields ensures that there are no checksum fields in the
+    given dict that we do not know about.
+    """

 
     rejmsg = []
     hashes = map(lambda x: x[0], known_hashes)
 
     rejmsg = []
     hashes = map(lambda x: x[0], known_hashes)


@@ -334,9 +369,11 @@ def _ensure_changes_hash(changes, format, version, files, hashname, hashfunc):
 # access the checksums easily.
 
 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
 # access the checksums easily.
 
 def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):

-    """ensure_dsc_hashes' task is to ensure that each and every *present* hash
+    """
+    ensure_dsc_hashes' task is to ensure that each and every *present* hash

     in the dsc is correct, i.e. identical to the changes file and if necessary
     in the dsc is correct, i.e. identical to the changes file and if necessary

-    the pool.  The latter task is delegated to check_hash."""
+    the pool.  The latter task is delegated to check_hash.
+    """

 
     rejmsg = []
     if not dsc.has_key('Checksums-%s' % (hashname,)):
 
     rejmsg = []
     if not dsc.has_key('Checksums-%s' % (hashname,)):


@@ -349,64 +386,29 @@ def _ensure_dsc_hash(dsc, dsc_files, hashname, hashfunc):
 
 ################################################################################
 
 
 ################################################################################
 

-def ensure_hashes(changes, dsc, files, dsc_files):
-    rejmsg = []
-
-    # Make sure we recognise the format of the Files: field in the .changes
-    format = changes.get("format", "0.0").split(".", 1)
-    if len(format) == 2:
-        format = int(format[0]), int(format[1])
-    else:
-        format = int(float(format[0])), 0
-
-    # We need to deal with the original changes blob, as the fields we need
-    # might not be in the changes dict serialised into the .dak anymore.
-    orig_changes = parse_deb822(changes['filecontents'])
-
-    # Copy the checksums over to the current changes dict.  This will keep
-    # the existing modifications to it intact.
-    for field in orig_changes:
-        if field.startswith('checksums-'):
-            changes[field] = orig_changes[field]
-
-    # Check for unsupported hashes
-    rejmsg.extend(check_hash_fields(".changes", changes))
-    rejmsg.extend(check_hash_fields(".dsc", dsc))
-
-    # We have to calculate the hash if we have an earlier changes version than
-    # the hash appears in rather than require it exist in the changes file
-    for hashname, hashfunc, version in known_hashes:
-        rejmsg.extend(_ensure_changes_hash(changes, format, version, files,
-            hashname, hashfunc))
-        if "source" in changes["architecture"]:
-            rejmsg.extend(_ensure_dsc_hash(dsc, dsc_files, hashname,
-                hashfunc))
-
-    return rejmsg
-

 def parse_checksums(where, files, manifest, hashname):
     rejmsg = []
     field = 'checksums-%s' % hashname
     if not field in manifest:
         return rejmsg
 def parse_checksums(where, files, manifest, hashname):
     rejmsg = []
     field = 'checksums-%s' % hashname
     if not field in manifest:
         return rejmsg

-    input = manifest[field]
-    for line in input.split('\n'):
+    for line in manifest[field].split('\n'):

         if not line:
             break
         if not line:
             break

-        hash, size, file = line.strip().split(' ')
+        checksum, size, checkfile = line.strip().split(' ')
+        if not files.has_key(checkfile):

         # TODO: check for the file's entry in the original files dict, not
         # the one modified by (auto)byhand and other weird stuff
         # TODO: check for the file's entry in the original files dict, not
         # the one modified by (auto)byhand and other weird stuff

-        #if not files.has_key(file):

         #    rejmsg.append("%s: not present in files but in checksums-%s in %s" %
         #        (file, hashname, where))
         #    rejmsg.append("%s: not present in files but in checksums-%s in %s" %
         #        (file, hashname, where))

-        if not files[file]["size"] == size:
+            continue
+        if not files[checkfile]["size"] == size:

             rejmsg.append("%s: size differs for files and checksums-%s entry "\
             rejmsg.append("%s: size differs for files and checksums-%s entry "\

-                "in %s" % (file, hashname, where))
+                "in %s" % (checkfile, hashname, where))

             continue
             continue

-        files[file][hash_key(hashname)] = hash
+        files[checkfile][hash_key(hashname)] = checksum

     for f in files.keys():
         if not files[f].has_key(hash_key(hashname)):
     for f in files.keys():
         if not files[f].has_key(hash_key(hashname)):

-            rejmsg.append("%s: no entry in checksums-%s in %s" % (file,
+            rejmsg.append("%s: no entry in checksums-%s in %s" % (checkfile,

                 hashname, where))
     return rejmsg
 
                 hashname, where))
     return rejmsg
 


@@ -435,7 +437,10 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
         format = format[:2]
 
     if is_a_dsc:
         format = format[:2]
 
     if is_a_dsc:

-        if format != (1,0):
+        # format = (1,0) are the only formats we currently accept,
+        # format = (0,0) are missing format headers of which we still
+        # have some in the archive.
+        if format != (1,0) and format != (0,0):

             raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
     else:
         if (format < (1,5) or format > (1,8)):
             raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
     else:
         if (format < (1,5) or format > (1,8)):


@@ -474,94 +479,77 @@ def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):
 
 ################################################################################
 
 
 ################################################################################
 

-def force_to_utf8(s):
-    """Forces a string to UTF-8.  If the string isn't already UTF-8,
-it's assumed to be ISO-8859-1."""
-    try:
-        unicode(s, 'utf-8')
-        return s
-    except UnicodeError:
-        latin1_s = unicode(s,'iso8859-1')
-        return latin1_s.encode('utf-8')
-
-def rfc2047_encode(s):
-    """Encodes a (header) string per RFC2047 if necessary.  If the
-string is neither ASCII nor UTF-8, it's assumed to be ISO-8859-1."""
-    try:
-        codecs.lookup('ascii')[1](s)
-        return s
-    except UnicodeError:
-        pass
-    try:
-        codecs.lookup('utf-8')[1](s)
-        h = email.Header.Header(s, 'utf-8', 998)
-        return str(h)
-    except UnicodeError:
-        h = email.Header.Header(s, 'iso-8859-1', 998)
-        return str(h)
-
-################################################################################
-
-# <Culus> 'The standard sucks, but my tool is supposed to interoperate
-#          with it. I know - I'll fix the suckage and make things
-#          incompatible!'
-
-def fix_maintainer (maintainer):
-    """Parses a Maintainer or Changed-By field and returns:
-  (1) an RFC822 compatible version,
-  (2) an RFC2047 compatible version,
-  (3) the name
-  (4) the email
-
-The name is forced to UTF-8 for both (1) and (3).  If the name field
-contains '.' or ',' (as allowed by Debian policy), (1) and (2) are
-switched to 'email (name)' format."""
-    maintainer = maintainer.strip()
-    if not maintainer:
-        return ('', '', '', '')
-
-    if maintainer.find("<") == -1:
-        email = maintainer
-        name = ""
-    elif (maintainer[0] == "<" and maintainer[-1:] == ">"):
-        email = maintainer[1:-1]
-        name = ""
-    else:
-        m = re_parse_maintainer.match(maintainer)
-        if not m:
-            raise ParseMaintError, "Doesn't parse as a valid Maintainer field."
-        name = m.group(1)
-        email = m.group(2)
-
-    # Get an RFC2047 compliant version of the name
-    rfc2047_name = rfc2047_encode(name)
-
-    # Force the name to be UTF-8
-    name = force_to_utf8(name)
-
-    if name.find(',') != -1 or name.find('.') != -1:
-        rfc822_maint = "%s (%s)" % (email, name)
-        rfc2047_maint = "%s (%s)" % (email, rfc2047_name)
-    else:
-        rfc822_maint = "%s <%s>" % (name, email)
-        rfc2047_maint = "%s <%s>" % (rfc2047_name, email)
-
-    if email.find("@") == -1 and email.find("buildd_") != 0:
-        raise ParseMaintError, "No @ found in email address part."
-
-    return (rfc822_maint, rfc2047_maint, name, email)
-
-################################################################################
-
-# sendmail wrapper, takes _either_ a message string or a file as arguments

 def send_mail (message, filename=""):
 def send_mail (message, filename=""):

-        # If we've been passed a string dump it into a temporary file
+    """sendmail wrapper, takes _either_ a message string or a file as arguments"""
+
+    # If we've been passed a string dump it into a temporary file

     if message:
     if message:

-        filename = tempfile.mktemp()
-        fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700)
+        (fd, filename) = tempfile.mkstemp()

         os.write (fd, message)
         os.close (fd)
 
         os.write (fd, message)
         os.close (fd)
 

+    if Cnf.has_key("Dinstall::MailWhiteList") and \
+           Cnf["Dinstall::MailWhiteList"] != "":
+        message_in = open_file(filename)
+        message_raw = modemail.message_from_file(message_in)
+        message_in.close();
+
+        whitelist = [];
+        whitelist_in = open_file(Cnf["Dinstall::MailWhiteList"])
+        try:
+            for line in whitelist_in:
+                if not re_whitespace_comment.match(line):
+                    if re_re_mark.match(line):
+                        whitelist.append(re.compile(re_re_mark.sub("", line.strip(), 1)))
+                    else:
+                        whitelist.append(re.compile(re.escape(line.strip())))
+        finally:
+            whitelist_in.close()
+
+        # Fields to check.
+        fields = ["To", "Bcc", "Cc"]
+        for field in fields:
+            # Check each field
+            value = message_raw.get(field, None)
+            if value != None:
+                match = [];
+                for item in value.split(","):
+                    (rfc822_maint, rfc2047_maint, name, email) = fix_maintainer(item.strip())
+                    mail_whitelisted = 0
+                    for wr in whitelist:
+                        if wr.match(email):
+                            mail_whitelisted = 1
+                            break
+                    if not mail_whitelisted:
+                        print "Skipping %s since it's not in %s" % (item, Cnf["Dinstall::MailWhiteList"])
+                        continue
+                    match.append(item)
+
+                # Doesn't have any mail in whitelist so remove the header
+                if len(match) == 0:
+                    del message_raw[field]
+                else:
+                    message_raw.replace_header(field, string.join(match, ", "))
+
+        # Change message fields in order if we don't have a To header
+        if not message_raw.has_key("To"):
+            fields.reverse()
+            for field in fields:
+                if message_raw.has_key(field):
+                    message_raw[fields[-1]] = message_raw[field]
+                    del message_raw[field]
+                    break
+            else:
+                # Clean up any temporary files
+                # and return, as we removed all recipients.
+                if message:
+                    os.unlink (filename);
+                return;
+
+        fd = os.open(filename, os.O_RDWR|os.O_EXCL, 0700);
+        os.write (fd, message_raw.as_string(True));
+        os.close (fd);
+

     # Invoke sendmail
     (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
     if (result != 0):
     # Invoke sendmail
     (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
     if (result != 0):


@@ -640,6 +628,15 @@ def where_am_i ():
 
 def which_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
 
 def which_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())

+    # In case we allow local config files per user, try if one exists
+    if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
+        homedir = os.getenv("HOME")
+        confpath = os.path.join(homedir, "/etc/dak.conf")
+        if os.path.exists(confpath):
+            apt_pkg.ReadConfigFileISC(Cnf,default_config)
+
+    # We are still in here, so there is no local config file or we do
+    # not allow local files. Do the normal stuff.

     if Cnf.get("Config::" + res[0] + "::DakConfig"):
         return Cnf["Config::" + res[0] + "::DakConfig"]
     else:
     if Cnf.get("Config::" + res[0] + "::DakConfig"):
         return Cnf["Config::" + res[0] + "::DakConfig"]
     else:


@@ -647,6 +644,13 @@ def which_conf_file ():
 
 def which_apt_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
 
 def which_apt_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())

+    # In case we allow local config files per user, try if one exists
+    if Cnf.FindB("Config::" + res[0] + "::AllowLocalConfig"):
+        homedir = os.getenv("HOME")
+        confpath = os.path.join(homedir, "/etc/dak.conf")
+        if os.path.exists(confpath):
+            apt_pkg.ReadConfigFileISC(Cnf,default_config)
+

     if Cnf.get("Config::" + res[0] + "::AptConfig"):
         return Cnf["Config::" + res[0] + "::AptConfig"]
     else:
     if Cnf.get("Config::" + res[0] + "::AptConfig"):
         return Cnf["Config::" + res[0] + "::AptConfig"]
     else:


@@ -662,23 +666,13 @@ def which_alias_file():
 
 ################################################################################
 
 
 ################################################################################
 

-# Escape characters which have meaning to SQL's regex comparison operator ('~')
-# (woefully incomplete)
-
-def regex_safe (s):
-    s = s.replace('+', '\\\\+')
-    s = s.replace('.', '\\\\.')
-    return s
-
-################################################################################
-
-# Perform a substition of template

 def TemplateSubst(map, filename):
 def TemplateSubst(map, filename):

-    file = open_file(filename)
-    template = file.read()
+    """ Perform a substition of template """
+    templatefile = open_file(filename)
+    template = templatefile.read()

     for x in map.keys():
         template = template.replace(x,map[x])
     for x in map.keys():
         template = template.replace(x,map[x])

-    file.close()
+    templatefile.close()

     return template
 
 ################################################################################
     return template
 
 ################################################################################


@@ -697,6 +691,9 @@ def warn(msg):
 def whoami ():
     return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
 
 def whoami ():
     return pwd.getpwuid(os.getuid())[4].split(',')[0].replace('.', '')
 

+def getusername ():
+    return pwd.getpwuid(os.getuid())[0]
+

 ################################################################################
 
 def size_type (c):
 ################################################################################
 
 def size_type (c):


@@ -719,8 +716,8 @@ def cc_fix_changes (changes):
     for j in o.split():
         changes["architecture"][j] = 1
 
     for j in o.split():
         changes["architecture"][j] = 1
 

-# Sort by source name, source version, 'have source', and then by filename

 def changes_compare (a, b):
 def changes_compare (a, b):

+    """ Sort by source name, source version, 'have source', and then by filename """

     try:
         a_changes = parse_changes(a)
     except:
     try:
         a_changes = parse_changes(a)
     except:


@@ -774,13 +771,13 @@ def find_next_free (dest, too_many=100):
 ################################################################################
 
 def result_join (original, sep = '\t'):
 ################################################################################
 
 def result_join (original, sep = '\t'):

-    list = []
+    resultlist = []

     for i in xrange(len(original)):
         if original[i] == None:
     for i in xrange(len(original)):
         if original[i] == None:

-            list.append("")
+            resultlist.append("")

         else:
         else:

-            list.append(original[i])
-    return sep.join(list)
+            resultlist.append(original[i])
+    return sep.join(resultlist)

 
 ################################################################################
 
 
 ################################################################################
 


@@ -798,18 +795,20 @@ def prefix_multi_line_string(str, prefix, include_blank_lines=0):
 ################################################################################
 
 def validate_changes_file_arg(filename, require_changes=1):
 ################################################################################
 
 def validate_changes_file_arg(filename, require_changes=1):

-    """'filename' is either a .changes or .dak file.  If 'filename' is a
-.dak file, it's changed to be the corresponding .changes file.  The
-function then checks if the .changes file a) exists and b) is
-readable and returns the .changes filename if so.  If there's a
-problem, the next action depends on the option 'require_changes'
-argument:
-
- o If 'require_changes' == -1, errors are ignored and the .changes
-                               filename is returned.
- o If 'require_changes' == 0, a warning is given and 'None' is returned.
- o If 'require_changes' == 1, a fatal error is raised.
-"""
+    """
+    'filename' is either a .changes or .dak file.  If 'filename' is a
+    .dak file, it's changed to be the corresponding .changes file.  The
+    function then checks if the .changes file a) exists and b) is
+    readable and returns the .changes filename if so.  If there's a
+    problem, the next action depends on the option 'require_changes'
+    argument:
+
+      - If 'require_changes' == -1, errors are ignored and the .changes
+        filename is returned.
+      - If 'require_changes' == 0, a warning is given and 'None' is returned.
+      - If 'require_changes' == 1, a fatal error is raised.
+
+    """

     error = None
 
     orig_filename = filename
     error = None
 
     orig_filename = filename


@@ -868,17 +867,21 @@ def get_conf():
 
 ################################################################################
 
 
 ################################################################################
 

-# Handle -a, -c and -s arguments; returns them as SQL constraints

 def parse_args(Options):
 def parse_args(Options):

+    """ Handle -a, -c and -s arguments; returns them as SQL constraints """
+    # XXX: This should go away and everything which calls it be converted
+    #      to use SQLA properly.  For now, we'll just fix it not to use
+    #      the old Pg interface though
+    session = DBConn().session()

     # Process suite
     if Options["Suite"]:
         suite_ids_list = []
     # Process suite
     if Options["Suite"]:
         suite_ids_list = []

-        for suite in split_args(Options["Suite"]):
-            suite_id = database.get_suite_id(suite)
-            if suite_id == -1:
-                warn("suite '%s' not recognised." % (suite))
+        for suitename in split_args(Options["Suite"]):
+            suite = get_suite(suitename, session=session)
+            if suite.suite_id is None:
+                warn("suite '%s' not recognised." % (suite.suite_name))

             else:
             else:

-                suite_ids_list.append(suite_id)
+                suite_ids_list.append(suite.suite_id)

         if suite_ids_list:
             con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
         else:
         if suite_ids_list:
             con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])
         else:


@@ -889,12 +892,12 @@ def parse_args(Options):
     # Process component
     if Options["Component"]:
         component_ids_list = []
     # Process component
     if Options["Component"]:
         component_ids_list = []

-        for component in split_args(Options["Component"]):
-            component_id = database.get_component_id(component)
-            if component_id == -1:
-                warn("component '%s' not recognised." % (component))
+        for componentname in split_args(Options["Component"]):
+            component = get_component(componentname, session=session)
+            if component is None:
+                warn("component '%s' not recognised." % (componentname))

             else:
             else:

-                component_ids_list.append(component_id)
+                component_ids_list.append(component.component_id)

         if component_ids_list:
             con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
         else:
         if component_ids_list:
             con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])
         else:


@@ -904,18 +907,18 @@ def parse_args(Options):
 
     # Process architecture
     con_architectures = ""
 
     # Process architecture
     con_architectures = ""

+    check_source = 0

     if Options["Architecture"]:
         arch_ids_list = []
     if Options["Architecture"]:
         arch_ids_list = []

-        check_source = 0
-        for architecture in split_args(Options["Architecture"]):
-            if architecture == "source":
+        for archname in split_args(Options["Architecture"]):
+            if archname == "source":

                 check_source = 1
             else:
                 check_source = 1
             else:

-                architecture_id = database.get_architecture_id(architecture)
-                if architecture_id == -1:
-                    warn("architecture '%s' not recognised." % (architecture))
+                arch = get_architecture(archname, session=session)
+                if arch is None:
+                    warn("architecture '%s' not recognised." % (archname))

                 else:
                 else:

-                    arch_ids_list.append(architecture_id)
+                    arch_ids_list.append(arch.arch_id)

         if arch_ids_list:
             con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
         else:
         if arch_ids_list:
             con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])
         else:


@@ -965,10 +968,13 @@ def try_with_debug(function):
 
 ################################################################################
 
 
 ################################################################################
 

-# Function for use in sorting lists of architectures.
-# Sorts normally except that 'source' dominates all others.
-

 def arch_compare_sw (a, b):
 def arch_compare_sw (a, b):

+    """
+    Function for use in sorting lists of architectures.
+
+    Sorts normally except that 'source' dominates all others.
+    """
+

     if a == "source" and b == "source":
         return 0
     elif a == "source":
     if a == "source" and b == "source":
         return 0
     elif a == "source":


@@ -980,13 +986,15 @@ def arch_compare_sw (a, b):
 
 ################################################################################
 
 
 ################################################################################
 

-# Split command line arguments which can be separated by either commas
-# or whitespace.  If dwim is set, it will complain about string ending
-# in comma since this usually means someone did 'dak ls -a i386, m68k
-# foo' or something and the inevitable confusion resulting from 'm68k'
-# being treated as an argument is undesirable.
-

 def split_args (s, dwim=1):
 def split_args (s, dwim=1):

+    """
+    Split command line arguments which can be separated by either commas
+    or whitespace.  If dwim is set, it will complain about string ending
+    in comma since this usually means someone did 'dak ls -a i386, m68k
+    foo' or something and the inevitable confusion resulting from 'm68k'
+    being treated as an argument is undesirable.
+    """
+

     if s.find(",") == -1:
         return s.split()
     else:
     if s.find(",") == -1:
         return s.split()
     else:


@@ -1000,9 +1008,12 @@ def Dict(**dict): return dict
 
 ########################################
 
 
 ########################################
 

-# Our very own version of commands.getouputstatus(), hacked to support
-# gpgv's status fd.

 def gpgv_get_status_output(cmd, status_read, status_write):
 def gpgv_get_status_output(cmd, status_read, status_write):

+    """
+    Our very own version of commands.getouputstatus(), hacked to support
+    gpgv's status fd.
+    """
+

     cmd = ['/bin/sh', '-c', cmd]
     p2cread, p2cwrite = os.pipe()
     c2pread, c2pwrite = os.pipe()
     cmd = ['/bin/sh', '-c', cmd]
     p2cread, p2cwrite = os.pipe()
     c2pread, c2pwrite = os.pipe()


@@ -1092,9 +1103,11 @@ def process_gpgv_output(status):
 ################################################################################
 
 def retrieve_key (filename, keyserver=None, keyring=None):
 ################################################################################
 
 def retrieve_key (filename, keyserver=None, keyring=None):

-    """Retrieve the key that signed 'filename' from 'keyserver' and
-add it to 'keyring'.  Returns nothing on success, or an error message
-on error."""
+    """
+    Retrieve the key that signed 'filename' from 'keyserver' and
+    add it to 'keyring'.  Returns nothing on success, or an error message
+    on error.
+    """

 
     # Defaults for keyserver and keyring
     if not keyserver:
 
     # Defaults for keyserver and keyring
     if not keyserver:


@@ -1107,7 +1120,7 @@ on error."""
         return "%s: tainted filename" % (filename)
 
     # Invoke gpgv on the file
         return "%s: tainted filename" % (filename)
 
     # Invoke gpgv on the file

-    status_read, status_write = os.pipe();
+    status_read, status_write = os.pipe()

     cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
     (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
 
     cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
     (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)
 


@@ -1143,28 +1156,32 @@ def gpg_keyring_args(keyrings=None):
 
 ################################################################################
 
 
 ################################################################################
 

-def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None):
-    """Check the signature of a file and return the fingerprint if the
-signature is valid or 'None' if it's not.  The first argument is the
-filename whose signature should be checked.  The second argument is a
-reject function and is called when an error is found.  The reject()
-function must allow for two arguments: the first is the error message,
-the second is an optional prefix string.  It's possible for reject()
-to be called more than once during an invocation of check_signature().
-The third argument is optional and is the name of the files the
-detached signature applies to.  The fourth argument is optional and is
-a *list* of keyrings to use.  'autofetch' can either be None, True or
-False.  If None, the default behaviour specified in the config will be
-used."""
+def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=None):
+    """
+    Check the signature of a file and return the fingerprint if the
+    signature is valid or 'None' if it's not.  The first argument is the
+    filename whose signature should be checked.  The second argument is a
+    reject function and is called when an error is found.  The reject()
+    function must allow for two arguments: the first is the error message,
+    the second is an optional prefix string.  It's possible for reject()
+    to be called more than once during an invocation of check_signature().
+    The third argument is optional and is the name of the files the
+    detached signature applies to.  The fourth argument is optional and is
+    a *list* of keyrings to use.  'autofetch' can either be None, True or
+    False.  If None, the default behaviour specified in the config will be
+    used.
+    """
+
+    rejects = []

 
     # Ensure the filename contains no shell meta-characters or other badness
     if not re_taint_free.match(sig_filename):
 
     # Ensure the filename contains no shell meta-characters or other badness
     if not re_taint_free.match(sig_filename):

-        reject("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
-        return None
+        rejects.append("!!WARNING!! tainted signature filename: '%s'." % (sig_filename))
+        return (None, rejects)

 
     if data_filename and not re_taint_free.match(data_filename):
 
     if data_filename and not re_taint_free.match(data_filename):

-        reject("!!WARNING!! tainted data filename: '%s'." % (data_filename))
-        return None
+        rejects.append("!!WARNING!! tainted data filename: '%s'." % (data_filename))
+        return (None, rejects)

 
     if not keyrings:
         keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
 
     if not keyrings:
         keyrings = Cnf.ValueList("Dinstall::GPGKeyring")


@@ -1175,11 +1192,11 @@ used."""
     if autofetch:
         error_msg = retrieve_key(sig_filename)
         if error_msg:
     if autofetch:
         error_msg = retrieve_key(sig_filename)
         if error_msg:

-            reject(error_msg)
-            return None
+            rejects.append(error_msg)
+            return (None, rejects)

 
     # Build the command line
 
     # Build the command line

-    status_read, status_write = os.pipe();
+    status_read, status_write = os.pipe()

     cmd = "gpgv --status-fd %s %s %s %s" % (
         status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
 
     cmd = "gpgv --status-fd %s %s %s %s" % (
         status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
 


@@ -1191,70 +1208,71 @@ used."""
 
     # If we failed to parse the status-fd output, let's just whine and bail now
     if internal_error:
 
     # If we failed to parse the status-fd output, let's just whine and bail now
     if internal_error:

-        reject("internal error while performing signature check on %s." % (sig_filename))
-        reject(internal_error, "")
-        reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
-        return None
+        rejects.append("internal error while performing signature check on %s." % (sig_filename))
+        rejects.append(internal_error, "")
+        rejects.append("Please report the above errors to the Archive maintainers by replying to this mail.", "")
+        return (None, rejects)

 
 

-    bad = ""

     # Now check for obviously bad things in the processed output
     if keywords.has_key("KEYREVOKED"):
     # Now check for obviously bad things in the processed output
     if keywords.has_key("KEYREVOKED"):

-        reject("The key used to sign %s has been revoked." % (sig_filename))
-        bad = 1
+        rejects.append("The key used to sign %s has been revoked." % (sig_filename))

     if keywords.has_key("BADSIG"):
     if keywords.has_key("BADSIG"):

-        reject("bad signature on %s." % (sig_filename))
-        bad = 1
+        rejects.append("bad signature on %s." % (sig_filename))

     if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):
     if keywords.has_key("ERRSIG") and not keywords.has_key("NO_PUBKEY"):

-        reject("failed to check signature on %s." % (sig_filename))
-        bad = 1
+        rejects.append("failed to check signature on %s." % (sig_filename))

     if keywords.has_key("NO_PUBKEY"):
         args = keywords["NO_PUBKEY"]
         if len(args) >= 1:
             key = args[0]
     if keywords.has_key("NO_PUBKEY"):
         args = keywords["NO_PUBKEY"]
         if len(args) >= 1:
             key = args[0]

-        reject("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))
-        bad = 1
+        rejects.append("The key (0x%s) used to sign %s wasn't found in the keyring(s)." % (key, sig_filename))

     if keywords.has_key("BADARMOR"):
     if keywords.has_key("BADARMOR"):

-        reject("ASCII armour of signature was corrupt in %s." % (sig_filename))
-        bad = 1
+        rejects.append("ASCII armour of signature was corrupt in %s." % (sig_filename))

     if keywords.has_key("NODATA"):
     if keywords.has_key("NODATA"):

-        reject("no signature found in %s." % (sig_filename))
-        bad = 1
+        rejects.append("no signature found in %s." % (sig_filename))
+    if keywords.has_key("EXPKEYSIG"):
+        args = keywords["EXPKEYSIG"]
+        if len(args) >= 1:
+            key = args[0]
+        rejects.append("Signature made by expired key 0x%s" % (key))

     if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
         args = keywords["KEYEXPIRED"]
     if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
         args = keywords["KEYEXPIRED"]

+        expiredate=""

         if len(args) >= 1:
         if len(args) >= 1:

-            key = args[0]
-        reject("The key (0x%s) used to sign %s has expired." % (key, sig_filename))
-        bad = 1
+            timestamp = args[0]
+            if timestamp.count("T") == 0:
+                try:
+                    expiredate = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
+                except ValueError:
+                    expiredate = "unknown (%s)" % (timestamp)
+            else:
+                expiredate = timestamp
+        rejects.append("The key used to sign %s has expired on %s" % (sig_filename, expiredate))

 
 

-    if bad:
-        return None
+    if len(rejects) > 0:
+        return (None, rejects)

 
     # Next check gpgv exited with a zero return code
     if exit_status:
 
     # Next check gpgv exited with a zero return code
     if exit_status:

-        reject("gpgv failed while checking %s." % (sig_filename))
+        rejects.append("gpgv failed while checking %s." % (sig_filename))

         if status.strip():
         if status.strip():

-            reject(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")
+            rejects.append(prefix_multi_line_string(status, " [GPG status-fd output:] "), "")

         else:
         else:

-            reject(prefix_multi_line_string(output, " [GPG output:] "), "")
-        return None
+            rejects.append(prefix_multi_line_string(output, " [GPG output:] "), "")
+        return (None, rejects)

 
     # Sanity check the good stuff we expect
     if not keywords.has_key("VALIDSIG"):
 
     # Sanity check the good stuff we expect
     if not keywords.has_key("VALIDSIG"):

-        reject("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))
-        bad = 1
+        rejects.append("signature on %s does not appear to be valid [No VALIDSIG]." % (sig_filename))

     else:
         args = keywords["VALIDSIG"]
         if len(args) < 1:
     else:
         args = keywords["VALIDSIG"]
         if len(args) < 1:

-            reject("internal error while checking signature on %s." % (sig_filename))
-            bad = 1
+            rejects.append("internal error while checking signature on %s." % (sig_filename))

         else:
             fingerprint = args[0]
     if not keywords.has_key("GOODSIG"):
         else:
             fingerprint = args[0]
     if not keywords.has_key("GOODSIG"):

-        reject("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))
-        bad = 1
+        rejects.append("signature on %s does not appear to be valid [No GOODSIG]." % (sig_filename))

     if not keywords.has_key("SIG_ID"):
     if not keywords.has_key("SIG_ID"):

-        reject("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))
-        bad = 1
+        rejects.append("signature on %s does not appear to be valid [No SIG_ID]." % (sig_filename))

 
     # Finally ensure there's not something we don't recognise
     known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
 
     # Finally ensure there's not something we don't recognise
     known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",


@@ -1263,13 +1281,12 @@ used."""
 
     for keyword in keywords.keys():
         if not known_keywords.has_key(keyword):
 
     for keyword in keywords.keys():
         if not known_keywords.has_key(keyword):

-            reject("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))
-            bad = 1
+            rejects.append("found unknown status token '%s' from gpgv with args '%r' in %s." % (keyword, keywords[keyword], sig_filename))

 
 

-    if bad:
-        return None
+    if len(rejects) > 0:
+        return (None, rejects)

     else:
     else:

-        return fingerprint
+        return (fingerprint, [])

 
 ################################################################################
 
 
 ################################################################################
 


@@ -1325,9 +1342,11 @@ def wrap(paragraph, max_length, prefix=""):
 
 ################################################################################
 
 
 ################################################################################
 

-# Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
-# Returns fixed 'src'

 def clean_symlink (src, dest, root):
 def clean_symlink (src, dest, root):

+    """
+    Relativize an absolute symlink from 'src' -> 'dest' relative to 'root'.
+    Returns fixed 'src'
+    """

     src = src.replace(root, '', 1)
     dest = dest.replace(root, '', 1)
     dest = os.path.dirname(dest)
     src = src.replace(root, '', 1)
     dest = dest.replace(root, '', 1)
     dest = os.path.dirname(dest)


@@ -1336,32 +1355,36 @@ def clean_symlink (src, dest, root):
 
 ################################################################################
 
 
 ################################################################################
 

-def temp_filename(directory=None, dotprefix=None, perms=0700):
-    """Return a secure and unique filename by pre-creating it.
-If 'directory' is non-null, it will be the directory the file is pre-created in.
-If 'dotprefix' is non-null, the filename will be prefixed with a '.'."""
+def temp_filename(directory=None, prefix="dak", suffix=""):
+    """
+    Return a secure and unique filename by pre-creating it.
+    If 'directory' is non-null, it will be the directory the file is pre-created in.
+    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
+    If 'suffix' is non-null, the filename will end with it.

 
 

-    if directory:
-        old_tempdir = tempfile.tempdir
-        tempfile.tempdir = directory
+    Returns a pair (fd, name).
+    """

 
 

-    filename = tempfile.mktemp()
+    return tempfile.mkstemp(suffix, prefix, directory)

 
 

-    if dotprefix:
-        filename = "%s/.%s" % (os.path.dirname(filename), os.path.basename(filename))
-    fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, perms)
-    os.close(fd)
+################################################################################

 
 

-    if directory:
-        tempfile.tempdir = old_tempdir
+def temp_dirname(parent=None, prefix="dak", suffix=""):
+    """
+    Return a secure and unique directory by pre-creating it.
+    If 'parent' is non-null, it will be the directory the directory is pre-created in.
+    If 'prefix' is non-null, the filename will be prefixed with it, default is dak.
+    If 'suffix' is non-null, the filename will end with it.

 
 

-    return filename
+    Returns a pathname to the new directory
+    """

 
 

-################################################################################
+    return tempfile.mkdtemp(suffix, prefix, parent)

 
 

-# checks if the user part of the email is listed in the alias file
+################################################################################

 
 def is_email_alias(email):
 
 def is_email_alias(email):

+    """ checks if the user part of the email is listed in the alias file """

     global alias_cache
     if alias_cache == None:
         aliasfn = which_alias_file()
     global alias_cache
     if alias_cache == None:
         aliasfn = which_alias_file()


@@ -1374,6 +1397,25 @@ def is_email_alias(email):
 
 ################################################################################
 
 
 ################################################################################
 

+def get_changes_files(dir):
+    """
+    Takes a directory and lists all .changes files in it (as well as chdir'ing
+    to the directory; this is due to broken behaviour on the part of p-u/p-a
+    when you're not in the right place)
+
+    Returns a list of filenames
+    """
+    try:
+        # Much of the rest of p-u/p-a depends on being in the right place
+        os.chdir(dir)
+        changes_files = [x for x in os.listdir(dir) if x.endswith('.changes')]
+    except OSError, e:
+        fubar("Failed to read list from directory %s (%s)" % (dir, e))
+
+    return changes_files
+
+################################################################################
+

 apt_pkg.init()
 
 Cnf = apt_pkg.newConfiguration()
 apt_pkg.init()
 
 Cnf = apt_pkg.newConfiguration()


@@ -1382,4 +1424,4 @@ apt_pkg.ReadConfigFileISC(Cnf,default_config)
 if which_conf_file() != default_config:
     apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
 
 if which_conf_file() != default_config:
     apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
 

-################################################################################
+###############################################################################