Revert "use correct field names"

[dak.git] / daklib / utils.py

diff --git a/daklib/utils.py b/daklib/utils.py
old mode 100644 (file)
new mode 100755 (executable)
index c4c1366..7584524
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -2,7 +2,6 @@
 
 # Utility functions
 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>
 
 # Utility functions
 # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006  James Troup <james@nocrew.org>

-# $Id: utils.py,v 1.73 2005-03-18 05:24:38 troup Exp $

 
 ################################################################################
 
 
 ################################################################################
 


@@ -23,9 +22,10 @@
 ################################################################################
 
 import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \
 ################################################################################
 
 import codecs, commands, email.Header, os, pwd, re, select, socket, shutil, \

-       string, sys, tempfile, traceback
+       sys, tempfile, traceback, stat

 import apt_pkg
 import database
 import apt_pkg
 import database

+from dak_exceptions import *

 
 ################################################################################
 
 
 ################################################################################
 


@@ -42,45 +42,30 @@ re_multi_line_field = re.compile(r"^\s(.*)")
 re_taint_free = re.compile(r"^[-+~/\.\w]+$")
 
 re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")
 re_taint_free = re.compile(r"^[-+~/\.\w]+$")
 
 re_parse_maintainer = re.compile(r"^\s*(\S.*\S)\s*\<([^\>]+)\>")

+re_gpg_uid = re.compile('^uid.*<([^>]*)>')

 
 

-changes_parse_error_exc = "Can't parse line in .changes file"
-invalid_dsc_format_exc = "Invalid .dsc file"
-nk_format_exc = "Unknown Format: in .changes file"
-no_files_exc = "No Files: field in .dsc or .changes file."
-cant_open_exc = "Can't open file"
-unknown_hostname_exc = "Unknown hostname"
-cant_overwrite_exc = "Permission denied; can't overwrite existent file."
-file_exists_exc = "Destination file exists"
-sendmail_failed_exc = "Sendmail invocation failed"
-tried_too_hard_exc = "Tried too hard to find a free filename."
+re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$")
+re_verwithext = re.compile(r"^(\d+)(?:\.(\d+))(?:\s+\((\S+)\))?$")
+
+re_srchasver = re.compile(r"^(\S+)\s+\((\S+)\)$")

 
 default_config = "/etc/dak/dak.conf"
 default_apt_config = "/etc/dak/apt.conf"
 
 
 default_config = "/etc/dak/dak.conf"
 default_apt_config = "/etc/dak/apt.conf"
 

-################################################################################
-
-class Error(Exception):
-    """Base class for exceptions in this module."""
-    pass
-
-class ParseMaintError(Error):
-    """Exception raised for errors in parsing a maintainer field.
+alias_cache = None
+key_uid_email_cache = {}

 
 

-    Attributes:
-       message -- explanation of the error
-    """
-
-    def __init__(self, message):
-        self.args = message,
-        self.message = message
+# (hashname, function, earliest_changes_version)
+known_hashes = [("sha1", apt_pkg.sha1sum, (1, 8)),
+                ("sha256", apt_pkg.sha256sum, (1, 8))]

 
 ################################################################################
 
 def open_file(filename, mode='r'):
     try:
 
 ################################################################################
 
 def open_file(filename, mode='r'):
     try:

-       f = open(filename, mode)
+        f = open(filename, mode)

     except IOError:
     except IOError:

-        raise cant_open_exc, filename
+        raise CantOpenError, filename

     return f
 
 ################################################################################
     return f
 
 ################################################################################


@@ -98,30 +83,11 @@ def our_raw_input(prompt=""):
 
 ################################################################################
 
 
 ################################################################################
 

-def str_isnum (s):
-    for c in s:
-        if c not in string.digits:
-            return 0
-    return 1
-
-################################################################################
-

 def extract_component_from_section(section):
     component = ""
 
     if section.find('/') != -1:
         component = section.split('/')[0]
 def extract_component_from_section(section):
     component = ""
 
     if section.find('/') != -1:
         component = section.split('/')[0]

-    if component.lower() == "non-us" and section.find('/') != -1:
-        s = component + '/' + section.split('/')[1]
-        if Cnf.has_key("Component::%s" % s): # Avoid e.g. non-US/libs
-            component = s
-
-    if section.lower() == "non-us":
-        component = "non-US/main"
-
-    # non-US prefix is case insensitive
-    if component.lower()[:6] == "non-us":
-        component = "non-US"+component[6:]

 
     # Expand default component
     if component == "":
 
     # Expand default component
     if component == "":


@@ -129,8 +95,6 @@ def extract_component_from_section(section):
             component = section
         else:
             component = "main"
             component = section
         else:
             component = "main"

-    elif component == "non-US":
-        component = "non-US/main"

 
     return (section, component)
 
 
     return (section, component)
 


@@ -164,7 +128,7 @@ The rules for (signing_rules == 1)-mode are:
     lines = changes_in.readlines()
 
     if not lines:
     lines = changes_in.readlines()
 
     if not lines:

-       raise changes_parse_error_exc, "[Empty changes file]"
+        raise ParseChangesError, "[Empty changes file]"

 
     # Reindex by line number so we can easily verify the format of
     # .dsc files...
 
     # Reindex by line number so we can easily verify the format of
     # .dsc files...


@@ -186,10 +150,10 @@ The rules for (signing_rules == 1)-mode are:
             if signing_rules == 1:
                 index += 1
                 if index > num_of_lines:
             if signing_rules == 1:
                 index += 1
                 if index > num_of_lines:

-                    raise invalid_dsc_format_exc, index
+                    raise InvalidDscError, index

                 line = indexed_lines[index]
                 if not line.startswith("-----BEGIN PGP SIGNATURE"):
                 line = indexed_lines[index]
                 if not line.startswith("-----BEGIN PGP SIGNATURE"):

-                    raise invalid_dsc_format_exc, index
+                    raise InvalidDscError, index

                 inside_signature = 0
                 break
             else:
                 inside_signature = 0
                 break
             else:


@@ -210,7 +174,7 @@ The rules for (signing_rules == 1)-mode are:
         if slf:
             field = slf.groups()[0].lower()
             changes[field] = slf.groups()[1]
         if slf:
             field = slf.groups()[0].lower()
             changes[field] = slf.groups()[1]

-           first = 1
+            first = 1

             continue
         if line == " .":
             changes[field] += '\n'
             continue
         if line == " .":
             changes[field] += '\n'


@@ -218,56 +182,191 @@ The rules for (signing_rules == 1)-mode are:
         mlf = re_multi_line_field.match(line)
         if mlf:
             if first == -1:
         mlf = re_multi_line_field.match(line)
         if mlf:
             if first == -1:

-                raise changes_parse_error_exc, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)
+                raise ParseChangesError, "'%s'\n [Multi-line field continuing on from nothing?]" % (line)

             if first == 1 and changes[field] != "":
                 changes[field] += '\n'
             first = 0
             if first == 1 and changes[field] != "":
                 changes[field] += '\n'
             first = 0

-           changes[field] += mlf.groups()[0] + '\n'
+            changes[field] += mlf.groups()[0] + '\n'

             continue
             continue

-       error += line
+        error += line

 
     if signing_rules == 1 and inside_signature:
 
     if signing_rules == 1 and inside_signature:

-        raise invalid_dsc_format_exc, index
+        raise InvalidDscError, index

 
     changes_in.close()
     changes["filecontents"] = "".join(lines)
 
 
     changes_in.close()
     changes["filecontents"] = "".join(lines)
 

+    if changes.has_key("source"):
+        # Strip the source version in brackets from the source field,
+        # put it in the "source-version" field instead.
+        srcver = re_srchasver.search(changes["source"])
+        if srcver:
+            changes["source"] = srcver.group(1)
+            changes["source-version"] = srcver.group(2)
+

     if error:
     if error:

-       raise changes_parse_error_exc, error
+        raise ParseChangesError, error

 
     return changes
 
 ################################################################################
 
 
     return changes
 
 ################################################################################
 

+def create_hash (lfiles, key, testfn, basedict = None):
+    rejmsg = []
+    for f in lfiles.keys():
+        try:
+            file_handle = open_file(f)
+        except CantOpenError:
+            rejmsg.append("Could not open file %s for checksumming" % (f))
+
+        # Check hash
+        if basedict and basedict.has_key(f):
+            basedict[f]['%ssum' % key] = testfn(file_handle)
+        file_handle.close()
+
+    return rejmsg
+
+################################################################################
+
+def check_hash (where, lfiles, key, testfn, basedict = None):
+    rejmsg = []
+    if basedict:
+        for f in basedict.keys():
+            if f not in lfiles:
+                rejmsg.append("%s: no %s checksum" % (f, key))
+
+    for f in lfiles.keys():
+        if basedict and f not in basedict:
+            rejmsg.append("%s: extraneous entry in %s checksums" % (f, key))
+
+        try:
+            file_handle = open_file(f)
+        except CantOpenError:
+            continue
+
+        # Check hash
+        if testfn(file_handle) != lfiles[f][key]:
+            rejmsg.append("%s: %s check failed." % (f, key))
+        file_handle.close()
+        # Store the hashes for later use
+        if basedict:
+            basedict[f]['%ssum' % key] = lfiles[f][key]
+        # Check size
+        actual_size = os.stat(f)[stat.ST_SIZE]
+        size = int(lfiles[f]["size"])
+        if size != actual_size:
+            rejmsg.append("%s: actual file size (%s) does not match size (%s) in %s"
+                   % (f, actual_size, size, where))
+
+    return rejmsg
+
+################################################################################
+
+def ensure_hashes(changes, dsc, files, dsc_files):
+    # Make sure we recognise the format of the Files: field
+    format = changes.get("format", "0.0").split(".",1)
+    if len(format) == 2:
+        format = int(format[0]), int(format[1])
+    else:
+        format = int(float(format[0])), 0
+
+    rejmsg = []
+    for x in changes:
+        if x.startswith("checksum-"):
+            h = x.split("-",1)[1]
+            if h not in dict(known_hashes):
+                rejmsg.append("Unsupported checksum field in .changes" % (h))
+
+    for x in dsc:
+        if x.startswith("checksum-"):
+            h = x.split("-",1)[1]
+            if h not in dict(known_hashes):
+                rejmsg.append("Unsupported checksum field in .dsc" % (h))
+
+    # We have to calculate the hash if we have an earlier changes version than
+    # the hash appears in rather than require it exist in the changes file
+    # I hate backwards compatibility
+    for h,f,v in known_hashes:
+        try:
+            if format < v:
+                for m in create_hash(files, h, f, files):
+                    rejmsg.append(m)
+            else:
+                for m in check_hash(".changes %s" % (h), files, h, f, files):
+                    rejmsg.append(m)
+        except NoFilesFieldError:
+            rejmsg.append("No Checksums-%s: field in .changes" % (h))
+        except UnknownFormatError, format:
+            rejmsg.append("%s: unknown format of .changes" % (format))
+        except ParseChangesError, line:
+            rejmsg.append("parse error for Checksums-%s in .changes, can't grok: %s." % (h, line))
+
+        if "source" not in changes["architecture"]: continue
+
+        try:
+            if format < v:
+                for m in create_hash(dsc_files, h, f, dsc_files):
+                    rejmsg.append(m)
+            else:
+                for m in check_hash(".dsc %s" % (h), dsc_files, h, f, dsc_files):
+                    rejmsg.append(m)
+        except UnknownFormatError, format:
+            rejmsg.append("%s: unknown format of .dsc" % (format))
+        except NoFilesFieldError:
+            rejmsg.append("No Checksums-%s: field in .dsc" % (h))
+        except ParseChangesError, line:
+            rejmsg.append("parse error for Checksums-%s in .dsc, can't grok: %s." % (h, line))
+
+    return rejmsg
+
+################################################################################
+

 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
 
 # Dropped support for 1.4 and ``buggy dchanges 3.4'' (?!) compared to di.pl
 

-def build_file_list(changes, is_a_dsc=0):
+def build_file_list(changes, is_a_dsc=0, field="files", hashname="md5sum"):

     files = {}
 
     # Make sure we have a Files: field to parse...
     files = {}
 
     # Make sure we have a Files: field to parse...

-    if not changes.has_key("files"):
-       raise no_files_exc
+    if not changes.has_key(field):
+        raise NoFilesFieldError

 
     # Make sure we recognise the format of the Files: field
 
     # Make sure we recognise the format of the Files: field

-    format = changes.get("format", "")
-    if format != "":
-       format = float(format)
-    if not is_a_dsc and (format < 1.5 or format > 2.0):
-       raise nk_format_exc, format
+    format = re_verwithext.search(changes.get("format", "0.0"))
+    if not format:
+        raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+
+    format = format.groups()
+    if format[1] == None:
+        format = int(float(format[0])), 0, format[2]
+    else:
+        format = int(format[0]), int(format[1]), format[2]
+    if format[2] == None:
+        format = format[:2]
+
+    if is_a_dsc:
+        if format != (1,0):
+            raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+    else:
+        if (format < (1,5) or format > (1,8)):
+            raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+        if field != "files" and format < (1,8):
+            raise UnknownFormatError, "%s" % (changes.get("format","0.0"))
+
+    includes_section = (not is_a_dsc) and field == "files"

 
     # Parse each entry/line:
 
     # Parse each entry/line:

-    for i in changes["files"].split('\n'):
+    for i in changes[field].split('\n'):

         if not i:
             break
         s = i.split()
         section = priority = ""
         try:
         if not i:
             break
         s = i.split()
         section = priority = ""
         try:

-            if is_a_dsc:
-                (md5, size, name) = s
-            else:
+            if includes_section:

                 (md5, size, section, priority, name) = s
                 (md5, size, section, priority, name) = s

+            else:
+                (md5, size, name) = s

         except ValueError:
         except ValueError:

-            raise changes_parse_error_exc, i
+            raise ParseChangesError, i

 
         if section == "":
             section = "-"
 
         if section == "":
             section = "-"


@@ -276,8 +375,9 @@ def build_file_list(changes, is_a_dsc=0):
 
         (section, component) = extract_component_from_section(section)
 
 
         (section, component) = extract_component_from_section(section)
 

-        files[name] = Dict(md5sum=md5, size=size, section=section,
+        files[name] = Dict(size=size, section=section,

                            priority=priority, component=component)
                            priority=priority, component=component)

+        files[name][hashname] = md5

 
     return files
 
 
     return files
 


@@ -364,48 +464,46 @@ switched to 'email (name)' format."""
 
 # sendmail wrapper, takes _either_ a message string or a file as arguments
 def send_mail (message, filename=""):
 
 # sendmail wrapper, takes _either_ a message string or a file as arguments
 def send_mail (message, filename=""):

-       # If we've been passed a string dump it into a temporary file
-       if message:
-            filename = tempfile.mktemp()
-            fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700)
-            os.write (fd, message)
-            os.close (fd)
-
-       # Invoke sendmail
-       (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
-       if (result != 0):
-            raise sendmail_failed_exc, output
-
-       # Clean up any temporary files
-       if message:
-            os.unlink (filename)
+        # If we've been passed a string dump it into a temporary file
+    if message:
+        filename = tempfile.mktemp()
+        fd = os.open(filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700)
+        os.write (fd, message)
+        os.close (fd)
+
+    # Invoke sendmail
+    (result, output) = commands.getstatusoutput("%s < %s" % (Cnf["Dinstall::SendmailCommand"], filename))
+    if (result != 0):
+        raise SendmailFailedError, output
+
+    # Clean up any temporary files
+    if message:
+        os.unlink (filename)

 
 ################################################################################
 
 def poolify (source, component):
     if component:
 
 ################################################################################
 
 def poolify (source, component):
     if component:

-       component += '/'
-    # FIXME: this is nasty
-    component = component.lower().replace("non-us/", "non-US/")
+        component += '/'

     if source[:3] == "lib":
     if source[:3] == "lib":

-       return component + source[:4] + '/' + source + '/'
+        return component + source[:4] + '/' + source + '/'

     else:
     else:

-       return component + source[:1] + '/' + source + '/'
+        return component + source[:1] + '/' + source + '/'

 
 ################################################################################
 
 def move (src, dest, overwrite = 0, perms = 0664):
     if os.path.exists(dest) and os.path.isdir(dest):
 
 ################################################################################
 
 def move (src, dest, overwrite = 0, perms = 0664):
     if os.path.exists(dest) and os.path.isdir(dest):

-       dest_dir = dest
+        dest_dir = dest

     else:
     else:

-       dest_dir = os.path.dirname(dest)
+        dest_dir = os.path.dirname(dest)

     if not os.path.exists(dest_dir):
     if not os.path.exists(dest_dir):

-       umask = os.umask(00000)
-       os.makedirs(dest_dir, 02775)
-       os.umask(umask)
+        umask = os.umask(00000)
+        os.makedirs(dest_dir, 02775)
+        os.umask(umask)

     #print "Moving %s to %s..." % (src, dest)
     if os.path.exists(dest) and os.path.isdir(dest):
     #print "Moving %s to %s..." % (src, dest)
     if os.path.exists(dest) and os.path.isdir(dest):

-       dest += '/' + os.path.basename(src)
+        dest += '/' + os.path.basename(src)

     # Don't overwrite unless forced to
     if os.path.exists(dest):
         if not overwrite:
     # Don't overwrite unless forced to
     if os.path.exists(dest):
         if not overwrite:


@@ -419,23 +517,23 @@ def move (src, dest, overwrite = 0, perms = 0664):
 
 def copy (src, dest, overwrite = 0, perms = 0664):
     if os.path.exists(dest) and os.path.isdir(dest):
 
 def copy (src, dest, overwrite = 0, perms = 0664):
     if os.path.exists(dest) and os.path.isdir(dest):

-       dest_dir = dest
+        dest_dir = dest

     else:
     else:

-       dest_dir = os.path.dirname(dest)
+        dest_dir = os.path.dirname(dest)

     if not os.path.exists(dest_dir):
     if not os.path.exists(dest_dir):

-       umask = os.umask(00000)
-       os.makedirs(dest_dir, 02775)
-       os.umask(umask)
+        umask = os.umask(00000)
+        os.makedirs(dest_dir, 02775)
+        os.umask(umask)

     #print "Copying %s to %s..." % (src, dest)
     if os.path.exists(dest) and os.path.isdir(dest):
     #print "Copying %s to %s..." % (src, dest)
     if os.path.exists(dest) and os.path.isdir(dest):

-       dest += '/' + os.path.basename(src)
+        dest += '/' + os.path.basename(src)

     # Don't overwrite unless forced to
     if os.path.exists(dest):
         if not overwrite:
     # Don't overwrite unless forced to
     if os.path.exists(dest):
         if not overwrite:

-            raise file_exists_exc
+            raise FileExistsError

         else:
             if not os.access(dest, os.W_OK):
         else:
             if not os.access(dest, os.W_OK):

-                raise cant_overwrite_exc
+                raise CantOverwriteError

     shutil.copy2(src, dest)
     os.chmod(dest, perms)
 
     shutil.copy2(src, dest)
     os.chmod(dest, perms)
 


@@ -445,23 +543,31 @@ def where_am_i ():
     res = socket.gethostbyaddr(socket.gethostname())
     database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
     if database_hostname:
     res = socket.gethostbyaddr(socket.gethostname())
     database_hostname = Cnf.get("Config::" + res[0] + "::DatabaseHostname")
     if database_hostname:

-       return database_hostname
+        return database_hostname

     else:
         return res[0]
 
 def which_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
     if Cnf.get("Config::" + res[0] + "::DakConfig"):
     else:
         return res[0]
 
 def which_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
     if Cnf.get("Config::" + res[0] + "::DakConfig"):

-       return Cnf["Config::" + res[0] + "::DakConfig"]
+        return Cnf["Config::" + res[0] + "::DakConfig"]

     else:
     else:

-       return default_config
+        return default_config

 
 def which_apt_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
     if Cnf.get("Config::" + res[0] + "::AptConfig"):
 
 def which_apt_conf_file ():
     res = socket.gethostbyaddr(socket.gethostname())
     if Cnf.get("Config::" + res[0] + "::AptConfig"):

-       return Cnf["Config::" + res[0] + "::AptConfig"]
+        return Cnf["Config::" + res[0] + "::AptConfig"]

     else:
     else:

-       return default_apt_config
+        return default_apt_config
+
+def which_alias_file():
+    hostname = socket.gethostbyaddr(socket.gethostname())[0]
+    aliasfn = '/var/lib/misc/'+hostname+'/forward-alias'
+    if os.path.exists(aliasfn):
+        return aliasfn
+    else:
+        return None

 
 ################################################################################
 
 
 ################################################################################
 


@@ -571,7 +677,7 @@ def find_next_free (dest, too_many=100):
         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:
         dest = orig_dest + '.' + repr(extra)
         extra += 1
     if extra >= too_many:

-        raise tried_too_hard_exc
+        raise NoFreeFilenameError

     return dest
 
 ################################################################################
     return dest
 
 ################################################################################


@@ -617,7 +723,7 @@ argument:
 
     orig_filename = filename
     if filename.endswith(".dak"):
 
     orig_filename = filename
     if filename.endswith(".dak"):

-        filename = filename[:-6]+".changes"
+        filename = filename[:-4]+".changes"

 
     if not filename.endswith(".changes"):
         error = "invalid file type; not a changes file"
 
     if not filename.endswith(".changes"):
         error = "invalid file type; not a changes file"


@@ -647,9 +753,9 @@ def real_arch(arch):
 ################################################################################
 
 def join_with_commas_and(list):
 ################################################################################
 
 def join_with_commas_and(list):

-       if len(list) == 0: return "nothing"
-       if len(list) == 1: return list[0]
-       return ", ".join(list[:-1]) + " and " + list[-1]
+    if len(list) == 0: return "nothing"
+    if len(list) == 1: return list[0]
+    return ", ".join(list[:-1]) + " and " + list[-1]

 
 ################################################################################
 
 
 ################################################################################
 


@@ -667,7 +773,7 @@ def pp_deps (deps):
 ################################################################################
 
 def get_conf():
 ################################################################################
 
 def get_conf():

-       return Cnf
+    return Cnf

 
 ################################################################################
 
 
 ################################################################################
 


@@ -683,7 +789,7 @@ def parse_args(Options):
             else:
                 suite_ids_list.append(suite_id)
         if suite_ids_list:
             else:
                 suite_ids_list.append(suite_id)
         if suite_ids_list:

-            con_suites = "AND su.id IN (%s)" % ", ".join(map(str, suite_ids_list))
+            con_suites = "AND su.id IN (%s)" % ", ".join([ str(i) for i in suite_ids_list ])

         else:
             fubar("No valid suite given.")
     else:
         else:
             fubar("No valid suite given.")
     else:


@@ -699,7 +805,7 @@ def parse_args(Options):
             else:
                 component_ids_list.append(component_id)
         if component_ids_list:
             else:
                 component_ids_list.append(component_id)
         if component_ids_list:

-            con_components = "AND c.id IN (%s)" % ", ".join(map(str, component_ids_list))
+            con_components = "AND c.id IN (%s)" % ", ".join([ str(i) for i in component_ids_list ])

         else:
             fubar("No valid component given.")
     else:
         else:
             fubar("No valid component given.")
     else:


@@ -720,7 +826,7 @@ def parse_args(Options):
                 else:
                     arch_ids_list.append(architecture_id)
         if arch_ids_list:
                 else:
                     arch_ids_list.append(architecture_id)
         if arch_ids_list:

-            con_architectures = "AND a.id IN (%s)" % ", ".join(map(str, arch_ids_list))
+            con_architectures = "AND a.id IN (%s)" % ", ".join([ str(i) for i in arch_ids_list ])

         else:
             if not check_source:
                 fubar("No valid architecture given.")
         else:
             if not check_source:
                 fubar("No valid architecture given.")


@@ -865,10 +971,88 @@ def gpgv_get_status_output(cmd, status_read, status_write):
 
     return output, status, exit_status
 
 
     return output, status, exit_status
 

-############################################################
+################################################################################

 
 

+def process_gpgv_output(status):
+    # Process the status-fd output
+    keywords = {}
+    internal_error = ""
+    for line in status.split('\n'):
+        line = line.strip()
+        if line == "":
+            continue
+        split = line.split()
+        if len(split) < 2:
+            internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
+            continue
+        (gnupg, keyword) = split[:2]
+        if gnupg != "[GNUPG:]":
+            internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
+            continue
+        args = split[2:]
+        if keywords.has_key(keyword) and keyword not in [ "NODATA", "SIGEXPIRED", "KEYEXPIRED" ]:
+            internal_error += "found duplicate status token ('%s').\n" % (keyword)
+            continue
+        else:
+            keywords[keyword] = args
+
+    return (keywords, internal_error)
+
+################################################################################
+
+def retrieve_key (filename, keyserver=None, keyring=None):
+    """Retrieve the key that signed 'filename' from 'keyserver' and
+add it to 'keyring'.  Returns nothing on success, or an error message
+on error."""
+
+    # Defaults for keyserver and keyring
+    if not keyserver:
+        keyserver = Cnf["Dinstall::KeyServer"]
+    if not keyring:
+        keyring = Cnf.ValueList("Dinstall::GPGKeyring")[0]
+
+    # Ensure the filename contains no shell meta-characters or other badness
+    if not re_taint_free.match(filename):
+        return "%s: tainted filename" % (filename)
+
+    # Invoke gpgv on the file
+    status_read, status_write = os.pipe();
+    cmd = "gpgv --status-fd %s --keyring /dev/null %s" % (status_write, filename)
+    (_, status, _) = gpgv_get_status_output(cmd, status_read, status_write)

 
 

-def check_signature (sig_filename, reject, data_filename="", keyrings=None):
+    # Process the status-fd output
+    (keywords, internal_error) = process_gpgv_output(status)
+    if internal_error:
+        return internal_error
+
+    if not keywords.has_key("NO_PUBKEY"):
+        return "didn't find expected NO_PUBKEY in gpgv status-fd output"
+
+    fingerprint = keywords["NO_PUBKEY"][0]
+    # XXX - gpg sucks.  You can't use --secret-keyring=/dev/null as
+    # it'll try to create a lockfile in /dev.  A better solution might
+    # be a tempfile or something.
+    cmd = "gpg --no-default-keyring --secret-keyring=%s --no-options" \
+          % (Cnf["Dinstall::SigningKeyring"])
+    cmd += " --keyring %s --keyserver %s --recv-key %s" \
+           % (keyring, keyserver, fingerprint)
+    (result, output) = commands.getstatusoutput(cmd)
+    if (result != 0):
+        return "'%s' failed with exit code %s" % (cmd, result)
+
+    return ""
+
+################################################################################
+
+def gpg_keyring_args(keyrings=None):
+    if not keyrings:
+        keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
+
+    return " ".join(["--keyring %s" % x for x in keyrings])
+
+################################################################################
+
+def check_signature (sig_filename, reject, data_filename="", keyrings=None, autofetch=None):

     """Check the signature of a file and return the fingerprint if the
 signature is valid or 'None' if it's not.  The first argument is the
 filename whose signature should be checked.  The second argument is a
     """Check the signature of a file and return the fingerprint if the
 signature is valid or 'None' if it's not.  The first argument is the
 filename whose signature should be checked.  The second argument is a


@@ -878,8 +1062,9 @@ the second is an optional prefix string.  It's possible for reject()
 to be called more than once during an invocation of check_signature().
 The third argument is optional and is the name of the files the
 detached signature applies to.  The fourth argument is optional and is
 to be called more than once during an invocation of check_signature().
 The third argument is optional and is the name of the files the
 detached signature applies to.  The fourth argument is optional and is

-a *list* of keyrings to use.
-"""
+a *list* of keyrings to use.  'autofetch' can either be None, True or
+False.  If None, the default behaviour specified in the config will be
+used."""

 
     # Ensure the filename contains no shell meta-characters or other badness
     if not re_taint_free.match(sig_filename):
 
     # Ensure the filename contains no shell meta-characters or other badness
     if not re_taint_free.match(sig_filename):


@@ -891,38 +1076,27 @@ a *list* of keyrings to use.
         return None
 
     if not keyrings:
         return None
 
     if not keyrings:

-        keyrings = (Cnf["Dinstall::PGPKeyring"], Cnf["Dinstall::GPGKeyring"])
+        keyrings = Cnf.ValueList("Dinstall::GPGKeyring")
+
+    # Autofetch the signing key if that's enabled
+    if autofetch == None:
+        autofetch = Cnf.get("Dinstall::KeyAutoFetch")
+    if autofetch:
+        error_msg = retrieve_key(sig_filename)
+        if error_msg:
+            reject(error_msg)
+            return None

 
     # Build the command line
 
     # Build the command line

-    status_read, status_write = os.pipe(); 
-    cmd = "gpgv --status-fd %s" % (status_write)
-    for keyring in keyrings:
-        cmd += " --keyring %s" % (keyring)
-    cmd += " %s %s" % (sig_filename, data_filename)
+    status_read, status_write = os.pipe();
+    cmd = "gpgv --status-fd %s %s %s %s" % (
+        status_write, gpg_keyring_args(keyrings), sig_filename, data_filename)
+

     # Invoke gpgv on the file
     (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
 
     # Process the status-fd output
     # Invoke gpgv on the file
     (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write)
 
     # Process the status-fd output

-    keywords = {}
-    bad = internal_error = ""
-    for line in status.split('\n'):
-        line = line.strip()
-        if line == "":
-            continue
-        split = line.split()
-        if len(split) < 2:
-            internal_error += "gpgv status line is malformed (< 2 atoms) ['%s'].\n" % (line)
-            continue
-        (gnupg, keyword) = split[:2]
-        if gnupg != "[GNUPG:]":
-            internal_error += "gpgv status line is malformed (incorrect prefix '%s').\n" % (gnupg)
-            continue
-        args = split[2:]
-        if keywords.has_key(keyword) and (keyword != "NODATA" and keyword != "SIGEXPIRED"):
-            internal_error += "found duplicate status token ('%s').\n" % (keyword)
-            continue
-        else:
-            keywords[keyword] = args
+    (keywords, internal_error) = process_gpgv_output(status)

 
     # If we failed to parse the status-fd output, let's just whine and bail now
     if internal_error:
 
     # If we failed to parse the status-fd output, let's just whine and bail now
     if internal_error:


@@ -931,10 +1105,8 @@ a *list* of keyrings to use.
         reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
         return None
 
         reject("Please report the above errors to the Archive maintainers by replying to this mail.", "")
         return None
 

+    bad = ""

     # Now check for obviously bad things in the processed output
     # Now check for obviously bad things in the processed output

-    if keywords.has_key("SIGEXPIRED"):
-        reject("The key used to sign %s has expired." % (sig_filename))
-        bad = 1

     if keywords.has_key("KEYREVOKED"):
         reject("The key used to sign %s has been revoked." % (sig_filename))
         bad = 1
     if keywords.has_key("KEYREVOKED"):
         reject("The key used to sign %s has been revoked." % (sig_filename))
         bad = 1


@@ -956,6 +1128,12 @@ a *list* of keyrings to use.
     if keywords.has_key("NODATA"):
         reject("no signature found in %s." % (sig_filename))
         bad = 1
     if keywords.has_key("NODATA"):
         reject("no signature found in %s." % (sig_filename))
         bad = 1

+    if keywords.has_key("KEYEXPIRED") and not keywords.has_key("GOODSIG"):
+        args = keywords["KEYEXPIRED"]
+        if len(args) >= 1:
+            key = args[0]
+        reject("The key (0x%s) used to sign %s has expired." % (key, sig_filename))
+        bad = 1

 
     if bad:
         return None
 
     if bad:
         return None


@@ -990,7 +1168,7 @@ a *list* of keyrings to use.
     # Finally ensure there's not something we don't recognise
     known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
                           SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
     # Finally ensure there's not something we don't recognise
     known_keywords = Dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
                           SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",

-                          NODATA="")
+                          NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="")

 
     for keyword in keywords.keys():
         if not known_keywords.has_key(keyword):
 
     for keyword in keywords.keys():
         if not known_keywords.has_key(keyword):


@@ -1004,6 +1182,25 @@ a *list* of keyrings to use.
 
 ################################################################################
 
 
 ################################################################################
 

+def gpg_get_key_addresses(fingerprint):
+    """retreive email addresses from gpg key uids for a given fingerprint"""
+    addresses = key_uid_email_cache.get(fingerprint)
+    if addresses != None:
+        return addresses
+    addresses = set()
+    cmd = "gpg --no-default-keyring %s --fingerprint %s" \
+                % (gpg_keyring_args(), fingerprint)
+    (result, output) = commands.getstatusoutput(cmd)
+    if result == 0:
+        for l in output.split('\n'):
+            m = re_gpg_uid.match(l)
+            if m:
+                addresses.add(m.group(1))
+    key_uid_email_cache[fingerprint] = addresses
+    return addresses
+
+################################################################################
+

 # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
 
 def wrap(paragraph, max_length, prefix=""):
 # Inspired(tm) by http://www.zopelabs.com/cookbook/1022242603
 
 def wrap(paragraph, max_length, prefix=""):


@@ -1071,12 +1268,27 @@ If 'dotprefix' is non-null, the filename will be prefixed with a '.'."""
 
 ################################################################################
 
 
 ################################################################################
 

+# checks if the user part of the email is listed in the alias file
+
+def is_email_alias(email):
+    global alias_cache
+    if alias_cache == None:
+        aliasfn = which_alias_file()
+        alias_cache = set()
+        if aliasfn:
+            for l in open(aliasfn):
+                alias_cache.add(l.split(':')[0])
+    uid = email.split('@')[0]
+    return uid in alias_cache
+
+################################################################################
+

 apt_pkg.init()
 
 Cnf = apt_pkg.newConfiguration()
 apt_pkg.ReadConfigFileISC(Cnf,default_config)
 
 if which_conf_file() != default_config:
 apt_pkg.init()
 
 Cnf = apt_pkg.newConfiguration()
 apt_pkg.ReadConfigFileISC(Cnf,default_config)
 
 if which_conf_file() != default_config:

-       apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())
+    apt_pkg.ReadConfigFileISC(Cnf,which_conf_file())

 
 ################################################################################
 
 ################################################################################