X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=daklib%2Fqueue.py;h=12e027190d32fb61bc98bf2ad2e3fb9c92c8f05e;hb=27e00376e81d1c37ff327ee0d39670b266418869;hp=dfbe36853490c8b9b99c72eebb50daa45d60c9de;hpb=87bcdbc7dffd6230cb4d49942fbd87ead6f2fffe;p=dak.git

diff --git a/daklib/queue.py b/daklib/queue.py
index dfbe3685..12e02719 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -51,19 +51,10 @@ from holding import Holding
 from urgencylog import UrgencyLog
 from dbconn import *
 from summarystats import SummaryStats
-from utils import parse_changes, check_dsc_files
+from utils import parse_changes, check_dsc_files, build_package_set
 from textutils import fix_maintainer
 from lintian import parse_lintian_output, generate_reject_messages
-
-# suppress some deprecation warnings in squeeze related to apt_pkg
-# module
-import warnings
-warnings.filterwarnings('ignore', \
-    "apt_pkg.ParseSection\(\) is deprecated. Please see apt_pkg\.TagSection\(\) for the replacement\.", \
-    DeprecationWarning)
-warnings.filterwarnings('ignore', \
-    "Attribute 'Find' of the 'apt_pkg\.TagSection' object is deprecated, use 'find' instead\.", \
-    DeprecationWarning)
+from contents import UnpackedSource
 
 ###############################################################################
 
@@ -86,6 +77,9 @@ def get_type(f, session):
         file_type = f["dbtype"]
     elif re_source_ext.match(f["type"]):
         file_type = "dsc"
+    elif f['architecture'] == 'source' and f["type"] == 'unreadable':
+        utils.warn('unreadable source file (will continue and hope for the best)')
+        return f["type"]
     else:
         file_type = f["type"]
         utils.fubar("invalid type (%s) for new.  Dazed, confused and sure as heck not continuing." % (file_type))
@@ -101,7 +95,7 @@ def get_type(f, session):
 
 # Determine what parts in a .changes are NEW
 
-def determine_new(filename, changes, files, warn=1, session = None):
+def determine_new(filename, changes, files, warn=1, session = None, dsc = None, new = None):
     """
     Determine what parts in a C{changes} file are NEW.
 
@@ -117,19 +111,32 @@ def determine_new(filename, changes, files, warn=1, session = None):
     @type warn: bool
     @param warn: Warn if overrides are added for (old)stable
 
+    @type dsc: Upload.Pkg.dsc dict
+    @param dsc: (optional); Dsc dictionary
+
+    @type new: dict
+    @param new: new packages as returned by a previous call to this function, but override information may have changed
+
     @rtype: dict
     @return: dictionary of NEW components.
 
     """
     # TODO: This should all use the database instead of parsing the changes
     # file again
-    new = {}
     byhand = {}
+    if new is None:
+        new = {}
 
     dbchg = get_dbchange(filename, session)
     if dbchg is None:
         print "Warning: cannot find changes file in database; won't check byhand"
 
+    # Try to get the Package-Set field from an included .dsc file (if possible).
+    if dsc:
+        for package, entry in build_package_set(dsc, session).items():
+            if not new.has_key(package):
+                new[package] = entry
+
     # Build up a list of potentially new things
     for name, f in files.items():
         # Keep a record of byhand elements
@@ -281,10 +288,10 @@ class TarTime(object):
         self.future_files = {}
         self.ancient_files = {}
 
-    def callback(self, Kind, Name, Link, Mode, UID, GID, Size, MTime, Major, Minor):
-        if MTime > self.future_cutoff:
+    def callback(self, member, data):
+        if member.mtime > self.future_cutoff:
             self.future_files[Name] = MTime
-        if MTime < self.past_cutoff:
+        if member.mtime < self.past_cutoff:
             self.ancient_files[Name] = MTime
 
 ###############################################################################
@@ -436,7 +443,8 @@ class Upload(object):
         cnf = Config()
         self.Subst = {}
         self.Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
-        self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
+        if cnf.has_key("Dinstall::BugServer"):
+            self.Subst["__BUG_SERVER__"] = cnf["Dinstall::BugServer"]
         self.Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
         self.Subst["__DAK_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"]
 
@@ -549,7 +557,7 @@ class Upload(object):
         except CantOpenError:
             self.rejects.append("%s: can't read file." % (filename))
             return False
-        except ParseChangesError, line:
+        except ParseChangesError as line:
             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
             return False
         except ChangesUnicodeError:
@@ -559,10 +567,10 @@ class Upload(object):
         # Parse the Files field from the .changes into another dictionary
         try:
             self.pkg.files.update(utils.build_file_list(self.pkg.changes))
-        except ParseChangesError, line:
+        except ParseChangesError as line:
             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
             return False
-        except UnknownFormatError, format:
+        except UnknownFormatError as format:
             self.rejects.append("%s: unknown format '%s'." % (filename, format))
             return False
 
@@ -600,7 +608,7 @@ class Upload(object):
              self.pkg.changes["maintainername"],
              self.pkg.changes["maintaineremail"]) = \
                    fix_maintainer (self.pkg.changes["maintainer"])
-        except ParseMaintError, msg:
+        except ParseMaintError as msg:
             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
                    % (filename, self.pkg.changes["maintainer"], msg))
 
@@ -611,7 +619,7 @@ class Upload(object):
              self.pkg.changes["changedbyname"],
              self.pkg.changes["changedbyemail"]) = \
                    fix_maintainer (self.pkg.changes.get("changed-by", ""))
-        except ParseMaintError, msg:
+        except ParseMaintError as msg:
             self.pkg.changes["changedby822"] = ""
             self.pkg.changes["changedby2047"] = ""
             self.pkg.changes["changedbyname"] = ""
@@ -692,7 +700,7 @@ class Upload(object):
 
         # Ensure target distributions exist
         for suite in self.pkg.changes["distribution"].keys():
-            if not Cnf.has_key("Suite::%s" % (suite)):
+            if not get_suite(suite.lower()):
                 self.rejects.append("Unknown distribution `%s'." % (suite))
 
     ###########################################################################
@@ -743,7 +751,7 @@ class Upload(object):
             self.rejects.append("%s: invalid version number '%s'." % (f, version))
 
         # Ensure the architecture of the .deb is one we know about.
-        default_suite = cnf.get("Dinstall::DefaultSuite", "Unstable")
+        default_suite = cnf.get("Dinstall::DefaultSuite", "unstable")
         architecture = control.Find("Architecture")
         upload_suite = self.pkg.changes["distribution"].keys()[0]
 
@@ -792,7 +800,7 @@ class Upload(object):
                         else:
                             entry["built-using"].append( (bu_so[0].source, bu_so[0].version, ) )
 
-            except ValueError, e:
+            except ValueError as e:
                 self.rejects.append("%s: Cannot parse Built-Using field: %s" % (f, str(e)))
 
 
@@ -871,15 +879,22 @@ class Upload(object):
                 # Check in one of the other directories
                 source_epochless_version = re_no_epoch.sub('', source_version)
                 dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
-                if os.path.exists(os.path.join(cnf["Dir::Queue::Byhand"], dsc_filename)):
+
+                byhand_dir = get_policy_queue('byhand', session).path
+                new_dir = get_policy_queue('new', session).path
+
+                if os.path.exists(os.path.join(byhand_dir, dsc_filename)):
                     entry["byhand"] = 1
-                elif os.path.exists(os.path.join(cnf["Dir::Queue::New"], dsc_filename)):
+                elif os.path.exists(os.path.join(new_dir, dsc_filename)):
                     entry["new"] = 1
                 else:
                     dsc_file_exists = False
-                    for myq in ["Embargoed", "Unembargoed", "ProposedUpdates", "OldProposedUpdates"]:
-                        if cnf.has_key("Dir::Queue::%s" % (myq)):
-                            if os.path.exists(os.path.join(cnf["Dir::Queue::" + myq], dsc_filename)):
+                    # TODO: Don't hardcode this list: use all relevant queues
+                    #       The question is how to determine what is relevant
+                    for queue_name in ["embargoed", "unembargoed", "proposedupdates", "oldproposedupdates"]:
+                        queue = get_policy_queue(queue_name, session)
+                        if queue:
+                            if os.path.exists(os.path.join(queue.path, dsc_filename)):
                                 dsc_file_exists = True
                                 break
 
@@ -953,8 +968,7 @@ class Upload(object):
                 entry["component"] = dest
 
         # Ensure the component is valid for the target suite
-        if cnf.has_key("Suite:%s::Components" % (suite)) and \
-           entry["component"] not in cnf.ValueList("Suite::%s::Components" % (suite)):
+        if entry["component"] not in get_component_names(session):
             self.rejects.append("unknown component `%s' for suite `%s'." % (entry["component"], suite))
             return
 
@@ -1032,7 +1046,7 @@ class Upload(object):
                    or (dbc.in_queue is not None
                        and dbc.in_queue.queue_name not in ["unchecked", "newstage"]):
                 self.rejects.append("%s file already known to dak" % base_filename)
-        except NoResultFound, e:
+        except NoResultFound as e:
             # not known, good
             pass
 
@@ -1041,10 +1055,11 @@ class Upload(object):
 
         for f, entry in self.pkg.files.items():
             # Ensure the file does not already exist in one of the accepted directories
-            for d in [ "Byhand", "New", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
-                if not cnf.has_key("Dir::Queue::%s" % (d)): continue
-                if os.path.exists(os.path.join(cnf["Dir::Queue::%s" % (d) ], f)):
-                    self.rejects.append("%s file already exists in the %s directory." % (f, d))
+            # TODO: Dynamically generate this list
+            for queue_name in [ "byhand", "new", "proposedupdates", "oldproposedupdates", "embargoed", "unembargoed" ]:
+                queue = get_policy_queue(queue_name, session)
+                if queue and os.path.exists(os.path.join(queue.path, f)):
+                    self.rejects.append("%s file already exists in the %s queue." % (f, queue_name))
 
             if not re_taint_free.match(f):
                 self.rejects.append("!!WARNING!! tainted filename: '%s'." % (f))
@@ -1100,44 +1115,83 @@ class Upload(object):
             if not has_source:
                 self.rejects.append("no source found and Architecture line in changes mention source.")
 
-            if not has_binaries and cnf.FindB("Dinstall::Reject::NoSourceOnly"):
+            if (not has_binaries) and (not cnf.FindB("Dinstall::AllowSourceOnlyUploads")):
                 self.rejects.append("source only uploads are not supported.")
 
     ###########################################################################
-    def check_dsc(self, action=True, session=None):
-        """Returns bool indicating whether or not the source changes are valid"""
-        # Ensure there is source to check
-        if not self.pkg.changes["architecture"].has_key("source"):
-            return True
 
-        # Find the .dsc
+    def __dsc_filename(self):
+        """
+        Returns: (Status, Dsc_Filename)
+        where
+          Status: Boolean; True when there was no error, False otherwise
+          Dsc_Filename: String; name of the dsc file if Status is True, reason for the error otherwise
+        """
         dsc_filename = None
-        for f, entry in self.pkg.files.items():
-            if entry["type"] == "dsc":
+
+        # find the dsc
+        for name, entry in self.pkg.files.items():
+            if entry.has_key("type") and entry["type"] == "dsc":
                 if dsc_filename:
-                    self.rejects.append("can not process a .changes file with multiple .dsc's.")
-                    return False
+                    return False, "cannot process a .changes file with multiple .dsc's."
                 else:
-                    dsc_filename = f
+                    dsc_filename = name
 
-        # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
         if not dsc_filename:
-            self.rejects.append("source uploads must contain a dsc file")
-            return False
+            return False, "source uploads must contain a dsc file"
+
+        return True, dsc_filename
+
+    def load_dsc(self, action=True, signing_rules=1):
+        """
+        Find and load the dsc from self.pkg.files into self.dsc
+
+        Returns: (Status, Reason)
+        where
+          Status: Boolean; True when there was no error, False otherwise
+          Reason: String; When Status is False this describes the error
+        """
+
+        # find the dsc
+        (status, dsc_filename) = self.__dsc_filename()
+        if not status:
+            # If status is false, dsc_filename has the reason
+            return False, dsc_filename
 
-        # Parse the .dsc file
         try:
-            self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=1, dsc_file=1))
+            self.pkg.dsc.update(utils.parse_changes(dsc_filename, signing_rules=signing_rules, dsc_file=1))
         except CantOpenError:
-            # if not -n copy_to_holding() will have done this for us...
             if not action:
-                self.rejects.append("%s: can't read file." % (dsc_filename))
-        except ParseChangesError, line:
-            self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
-        except InvalidDscError, line:
-            self.rejects.append("%s: syntax error on line %s." % (dsc_filename, line))
+                return False, "%s: can't read file." % (dsc_filename)
+        except ParseChangesError as line:
+            return False, "%s: parse error, can't grok: %s." % (dsc_filename, line)
+        except InvalidDscError as line:
+            return False, "%s: syntax error on line %s." % (dsc_filename, line)
         except ChangesUnicodeError:
-            self.rejects.append("%s: dsc file not proper utf-8." % (dsc_filename))
+            return False, "%s: dsc file not proper utf-8." % (dsc_filename)
+
+        return True, None
+
+    ###########################################################################
+
+    def check_dsc(self, action=True, session=None):
+        """Returns bool indicating whether or not the source changes are valid"""
+        # Ensure there is source to check
+        if not self.pkg.changes["architecture"].has_key("source"):
+            return True
+
+        if session is None:
+            session = DBConn().session()
+
+        (status, reason) = self.load_dsc(action=action)
+        if not status:
+            self.rejects.append(reason)
+            return False
+        (status, dsc_filename) = self.__dsc_filename()
+        if not status:
+            # If status is false, dsc_filename has the reason
+            self.rejects.append(dsc_filename)
+            return False
 
         # Build up the file list of files mentioned by the .dsc
         try:
@@ -1145,10 +1199,10 @@ class Upload(object):
         except NoFilesFieldError:
             self.rejects.append("%s: no Files: field." % (dsc_filename))
             return False
-        except UnknownFormatError, format:
+        except UnknownFormatError as format:
             self.rejects.append("%s: unknown format '%s'." % (dsc_filename, format))
             return False
-        except ParseChangesError, line:
+        except ParseChangesError as line:
             self.rejects.append("%s: parse error, can't grok: %s." % (dsc_filename, line))
             return False
 
@@ -1166,7 +1220,11 @@ class Upload(object):
 
         # Only a limited list of source formats are allowed in each suite
         for dist in self.pkg.changes["distribution"].keys():
-            allowed = [ x.format_name for x in get_suite_src_formats(dist, session) ]
+            suite = get_suite(dist, session=session)
+            if not suite:
+                self.rejects.append("%s: cannot find suite %s when checking source formats" % (dsc_filename, dist))
+                continue
+            allowed = [ x.format_name for x in suite.srcformats ]
             if self.pkg.dsc["format"] not in allowed:
                 self.rejects.append("%s: source format '%s' not allowed in %s (accepted: %s) " % (dsc_filename, self.pkg.dsc["format"], dist, ", ".join(allowed)))
 
@@ -1174,7 +1232,7 @@ class Upload(object):
         try:
             # We ignore the return value
             fix_maintainer(self.pkg.dsc["maintainer"])
-        except ParseMaintError, msg:
+        except ParseMaintError as msg:
             self.rejects.append("%s: Maintainer field ('%s') failed to parse: %s" \
                                  % (dsc_filename, self.pkg.dsc["maintainer"], msg))
 
@@ -1221,7 +1279,7 @@ class Upload(object):
 
         session.close()
 
-        return True
+        return (len(self.rejects) == 0)
 
     ###########################################################################
 
@@ -1265,14 +1323,13 @@ class Upload(object):
             os.symlink(self.pkg.orig_files[orig_file]["path"], dest)
 
         # Extract the source
-        cmd = "dpkg-source -sn -x %s" % (dsc_filename)
-        (result, output) = commands.getstatusoutput(cmd)
-        if (result != 0):
-            self.rejects.append("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result))
-            self.rejects.append(utils.prefix_multi_line_string(output, " [dpkg-source output:] "))
+        try:
+            unpacked = UnpackedSource(dsc_filename)
+        except Exception as e:
+            self.rejects.append("'dpkg-source -x' failed for %s. (%s)" % (dsc_filename, str(e)))
             return
 
-        if not cnf.Find("Dir::Queue::BTSVersionTrack"):
+        if not cnf.Find("Dir::BTSVersionTrack"):
             return
 
         # Get the upstream version
@@ -1281,19 +1338,19 @@ class Upload(object):
             upstr_version = re_strip_revision.sub('', upstr_version)
 
         # Ensure the changelog file exists
-        changelog_filename = "%s-%s/debian/changelog" % (self.pkg.dsc["source"], upstr_version)
-        if not os.path.exists(changelog_filename):
+        changelog_file = unpacked.get_changelog_file()
+        if changelog_file is None:
             self.rejects.append("%s: debian/changelog not found in extracted source." % (dsc_filename))
             return
 
         # Parse the changelog
         self.pkg.dsc["bts changelog"] = ""
-        changelog_file = utils.open_file(changelog_filename)
         for line in changelog_file.readlines():
             m = re_changelog_versions.match(line)
             if m:
                 self.pkg.dsc["bts changelog"] += line
         changelog_file.close()
+        unpacked.cleanup()
 
         # Check we found at least one revision in the changelog
         if not self.pkg.dsc["bts changelog"]:
@@ -1319,7 +1376,7 @@ class Upload(object):
 
         try:
             shutil.rmtree(tmpdir)
-        except OSError, e:
+        except OSError as e:
             if e.errno != errno.EACCES:
                 print "foobar"
                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
@@ -1332,7 +1389,7 @@ class Upload(object):
             if result != 0:
                 utils.fubar("'%s' failed with result %s." % (cmd, result))
             shutil.rmtree(tmpdir)
-        except Exception, e:
+        except Exception as e:
             print "foobar2 (%s)" % e
             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
 
@@ -1452,16 +1509,16 @@ class Upload(object):
                 continue
 
             # Look in some other queues for the file
-            queues = ('New', 'Byhand', 'ProposedUpdates',
-                'OldProposedUpdates', 'Embargoed', 'Unembargoed')
+            queue_names = ['new', 'byhand',
+                           'proposedupdates', 'oldproposedupdates',
+                           'embargoed', 'unembargoed']
 
-            for queue in queues:
-                if not cnf.get('Dir::Queue::%s' % queue):
+            for queue_name in queue_names:
+                queue = get_policy_queue(queue_name, session)
+                if not queue:
                     continue
 
-                queuefile_path = os.path.join(
-                    cnf['Dir::Queue::%s' % queue], filename
-                )
+                queuefile_path = os.path.join(queue.path, filename)
 
                 if not os.path.exists(queuefile_path):
                     # Does not exist in this queue
@@ -1495,7 +1552,7 @@ class Upload(object):
 
         # If we do not have a tagfile, don't do anything
         tagfile = cnf.get("Dinstall::LintianTags")
-        if tagfile is None:
+        if not tagfile:
             return
 
         # Parse the yaml file
@@ -1505,7 +1562,7 @@ class Upload(object):
 
         try:
             lintiantags = yaml.load(sourcecontent)['lintian']
-        except yaml.YAMLError, msg:
+        except yaml.YAMLError as msg:
             utils.fubar("Can not read the lintian tags file %s, YAML error: %s." % (tagfile, msg))
             return
 
@@ -1578,19 +1635,8 @@ class Upload(object):
             if entry["type"] == "deb":
                 tar.reset()
                 try:
-                    deb_file = utils.open_file(filename)
-                    apt_inst.debExtract(deb_file, tar.callback, "control.tar.gz")
-                    deb_file.seek(0)
-                    try:
-                        apt_inst.debExtract(deb_file, tar.callback, "data.tar.gz")
-                    except SystemError, e:
-                        # If we can't find a data.tar.gz, look for data.tar.bz2 instead.
-                        if not re.search(r"Cannot f[ui]nd chunk data.tar.gz$", str(e)):
-                            raise
-                        deb_file.seek(0)
-                        apt_inst.debExtract(deb_file,tar.callback,"data.tar.bz2")
-
-                    deb_file.close()
+                    deb = apt_inst.DebFile(filename)
+                    deb.control.go(tar.callback)
 
                     future_files = tar.future_files.keys()
                     if future_files:
@@ -1611,6 +1657,7 @@ class Upload(object):
                     self.rejects.append("%s: deb contents timestamp check failed [%s: %s]" % (filename, sys.exc_type, sys.exc_value))
 
     def check_if_upload_is_sponsored(self, uid_email, uid_name):
+        uid_email = '@'.join(uid_email.split('@')[:2])
         if uid_email in [self.pkg.changes["maintaineremail"], self.pkg.changes["changedbyemail"]]:
             sponsored = False
         elif uid_name in [self.pkg.changes["maintainername"], self.pkg.changes["changedbyname"]]:
@@ -1619,8 +1666,12 @@ class Upload(object):
                 sponsored = True
         else:
             sponsored = True
+            sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
+            debian_emails = filter(lambda addr: addr.endswith('@debian.org'), sponsor_addresses)
+            if uid_email not in debian_emails:
+                if debian_emails:
+                    uid_email = debian_emails[0]
             if ("source" in self.pkg.changes["architecture"] and uid_email and utils.is_email_alias(uid_email)):
-                sponsor_addresses = utils.gpg_get_key_addresses(self.pkg.changes["fingerprint"])
                 if (self.pkg.changes["maintaineremail"] not in sponsor_addresses and
                     self.pkg.changes["changedbyemail"] not in sponsor_addresses):
                         self.pkg.changes["sponsoremail"] = uid_email
@@ -1662,22 +1713,22 @@ class Upload(object):
         # Check any one-off upload blocks
         self.check_upload_blocks(fpr, session)
 
-        # Start with DM as a special case
+        # If the source_acl is None, source is never allowed
+        if fpr.source_acl is None:
+            if self.pkg.changes["architecture"].has_key("source"):
+                rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
+                rej += '\nPlease contact ftpmaster if you think this is incorrect'
+                self.rejects.append(rej)
+                return
+        # Do DM as a special case
         # DM is a special case unfortunately, so we check it first
         # (keys with no source access get more access than DMs in one
         #  way; DMs can only upload for their packages whether source
         #  or binary, whereas keys with no access might be able to
         #  upload some binaries)
-        if fpr.source_acl.access_level == 'dm':
+        elif fpr.source_acl.access_level == 'dm':
             self.check_dm_upload(fpr, session)
         else:
-            # Check source-based permissions for other types
-            if self.pkg.changes["architecture"].has_key("source") and \
-                fpr.source_acl.access_level is None:
-                rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
-                rej += '\nPlease contact ftpmaster if you think this is incorrect'
-                self.rejects.append(rej)
-                return
             # If not a DM, we allow full upload rights
             uid_email = "%s@debian.org" % (fpr.uid.uid)
             self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
@@ -1699,8 +1750,11 @@ class Upload(object):
 
         if len(tmparches.keys()) > 0:
             if fpr.binary_reject:
-                rej = ".changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
-                rej += "\narchitectures involved are: ", ",".join(tmparches.keys())
+                rej = "changes file contains files of architectures not permitted for fingerprint %s" % fpr.fingerprint
+                if len(tmparches.keys()) == 1:
+                    rej += "\n\narchitecture involved is: %s" % ",".join(tmparches.keys())
+                else:
+                    rej += "\n\narchitectures involved are: %s" % ",".join(tmparches.keys())
                 self.rejects.append(rej)
             else:
                 # TODO: This is where we'll implement reject vs throw away binaries later
@@ -1769,10 +1823,10 @@ class Upload(object):
         ## experimental lists the uploader in the Maintainer: or Uploaders: fields (ie,
         ## non-developer maintainers cannot NMU or hijack packages)
 
-        # srcuploaders includes the maintainer
+        # uploader includes the maintainer
         accept = False
-        for sup in r.srcuploaders:
-            (rfc822, rfc2047, name, email) = sup.maintainer.get_split_maintainer()
+        for uploader in r.uploaders:
+            (rfc822, rfc2047, name, email) = uploader.get_split_maintainer()
             # Eww - I hope we never have two people with the same name in Debian
             if email == fpr.uid.uid or name == fpr.uid.name:
                 accept = True
@@ -1805,7 +1859,7 @@ class Upload(object):
 
         # Also only check if there is a file defined (and existant) with
         # checks.
-        transpath = cnf.get("Dinstall::Reject::ReleaseTransitions", "")
+        transpath = cnf.get("Dinstall::ReleaseTransitions", "")
         if transpath == "" or not os.path.exists(transpath):
             return
 
@@ -1814,7 +1868,7 @@ class Upload(object):
         sourcecontent = sourcefile.read()
         try:
             transitions = yaml.load(sourcecontent)
-        except yaml.YAMLError, msg:
+        except yaml.YAMLError as msg:
             # This shouldn't happen, there is a wrapper to edit the file which
             # checks it, but we prefer to be safe than ending up rejecting
             # everything.
@@ -1958,26 +2012,31 @@ distribution."""
         """
 
         cnf = Config()
-        announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
+
+        # Skip all of this if not sending mail to avoid confusing people
+        if cnf.has_key("Dinstall::Options::No-Mail") and cnf["Dinstall::Options::No-Mail"]:
+            return ""
 
         # Only do announcements for source uploads with a recent dpkg-dev installed
         if float(self.pkg.changes.get("format", 0)) < 1.6 or not \
            self.pkg.changes["architecture"].has_key("source"):
             return ""
 
-        lists_done = {}
-        summary = ""
+        announcetemplate = os.path.join(cnf["Dir::Templates"], 'process-unchecked.announce')
 
-        self.Subst["__SHORT_SUMMARY__"] = short_summary
+        lists_todo = {}
+        summary = ""
 
+        # Get a unique list of target lists
         for dist in self.pkg.changes["distribution"].keys():
             suite = get_suite(dist)
             if suite is None: continue
-            announce_list = suite.announce
-            if announce_list == "" or lists_done.has_key(announce_list):
-                continue
+            for tgt in suite.announce:
+                lists_todo[tgt] = 1
 
-            lists_done[announce_list] = 1
+        self.Subst["__SHORT_SUMMARY__"] = short_summary
+
+        for announce_list in lists_todo.keys():
             summary += "Announcing to %s\n" % (announce_list)
 
             if action:
@@ -1993,7 +2052,7 @@ distribution."""
 
                 del self.Subst["__ANNOUNCE_LIST_ADDRESS__"]
 
-        if cnf.FindB("Dinstall::CloseBugs"):
+        if cnf.FindB("Dinstall::CloseBugs") and cnf.has_key("Dinstall::BugServer"):
             summary = self.close_bugs(summary, action)
 
         del self.Subst["__SHORT_SUMMARY__"]
@@ -2025,6 +2084,7 @@ distribution."""
         print "Installing."
         self.logger.log(["installing changes", self.pkg.changes_file])
 
+        binaries = []
         poolfiles = []
 
         # Add the .dsc file to the DB first
@@ -2037,7 +2097,9 @@ distribution."""
         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
         for newfile, entry in self.pkg.files.items():
             if entry["type"] == "deb":
-                poolfiles.append(add_deb_to_db(self, newfile, session))
+                b, pf = add_deb_to_db(self, newfile, session)
+                binaries.append(b)
+                poolfiles.append(pf)
 
         # If this is a sourceful diff only upload that is moving
         # cross-component we need to copy the .orig files into the new
@@ -2122,11 +2184,28 @@ distribution."""
         # Our SQL session will automatically start a new transaction after
         # the last commit
 
+        # Now ensure that the metadata has been added
+        # This has to be done after we copy the files into the pool
+        # For source if we have it:
+        if self.pkg.changes["architecture"].has_key("source"):
+            import_metadata_into_db(source, session)
+
+        # Now for any of our binaries
+        for b in binaries:
+            import_metadata_into_db(b, session)
+
+        session.commit()
+
         # Move the .changes into the 'done' directory
+        ye, mo, da = time.gmtime()[0:3]
+        donedir = os.path.join(cnf["Dir::Done"], str(ye), "%0.2d" % mo, "%0.2d" % da)
+        if not os.path.isdir(donedir):
+            os.makedirs(donedir)
+
         utils.move(self.pkg.changes_file,
-                   os.path.join(cnf["Dir::Queue::Done"], os.path.basename(self.pkg.changes_file)))
+                   os.path.join(donedir, os.path.basename(self.pkg.changes_file)))
 
-        if self.pkg.changes["architecture"].has_key("source") and cnf.get("Dir::UrgencyLog"):
+        if self.pkg.changes["architecture"].has_key("source"):
             UrgencyLog().log(self.pkg.dsc["source"], self.pkg.dsc["version"], self.pkg.changes["urgency"])
 
         self.update_subst()
@@ -2137,19 +2216,19 @@ distribution."""
         self.announce(short_summary, 1)
 
         ## Helper stuff for DebBugs Version Tracking
-        if cnf.Find("Dir::Queue::BTSVersionTrack"):
+        if cnf.Find("Dir::BTSVersionTrack"):
             if self.pkg.changes["architecture"].has_key("source"):
-                (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+                (fd, temp_filename) = utils.temp_filename(cnf["Dir::BTSVersionTrack"], prefix=".")
                 version_history = os.fdopen(fd, 'w')
                 version_history.write(self.pkg.dsc["bts changelog"])
                 version_history.close()
-                filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+                filename = "%s/%s" % (cnf["Dir::BTSVersionTrack"],
                                       self.pkg.changes_file[:-8]+".versions")
                 os.rename(temp_filename, filename)
                 os.chmod(filename, 0644)
 
             # Write out the binary -> source mapping.
-            (fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
+            (fd, temp_filename) = utils.temp_filename(cnf["Dir::BTSVersionTrack"], prefix=".")
             debinfo = os.fdopen(fd, 'w')
             for name, entry in sorted(self.pkg.files.items()):
                 if entry["type"] == "deb":
@@ -2158,7 +2237,7 @@ distribution."""
                                      entry["source version"]])
                     debinfo.write(line+"\n")
             debinfo.close()
-            filename = "%s/%s" % (cnf["Dir::Queue::BTSVersionTrack"],
+            filename = "%s/%s" % (cnf["Dir::BTSVersionTrack"],
                                   self.pkg.changes_file[:-8]+".debinfo")
             os.rename(temp_filename, filename)
             os.chmod(filename, 0644)
@@ -2260,11 +2339,11 @@ distribution."""
             if os.access(file_entry, os.R_OK) == 0:
                 continue
 
-            dest_file = os.path.join(cnf["Dir::Queue::Reject"], file_entry)
+            dest_file = os.path.join(cnf["Dir::Reject"], file_entry)
 
             try:
                 dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
-            except OSError, e:
+            except OSError as e:
                 # File exists?  Let's find a new name by adding a number
                 if e.errno == errno.EEXIST:
                     try:
@@ -2272,13 +2351,13 @@ distribution."""
                     except NoFreeFilenameError:
                         # Something's either gone badly Pete Tong, or
                         # someone is trying to exploit us.
-                        utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Queue::Reject"]))
+                        utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Reject"]))
                         return
 
                     # Make sure we really got it
                     try:
                         dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
-                    except OSError, e:
+                    except OSError as e:
                         # Likewise
                         utils.warn("**WARNING** failed to claim %s in the reject directory." % (file_entry))
                         return
@@ -2342,7 +2421,7 @@ distribution."""
         cnf = Config()
 
         reason_filename = self.pkg.changes_file[:-8] + ".reason"
-        reason_filename = os.path.join(cnf["Dir::Queue::Reject"], reason_filename)
+        reason_filename = os.path.join(cnf["Dir::Reject"], reason_filename)
 
         # Move all the files into the reject directory
         reject_files = self.pkg.files.keys() + [self.pkg.changes_file]
@@ -2386,6 +2465,8 @@ distribution."""
         if self.logger:
             self.logger.log(["rejected", self.pkg.changes_file])
 
+        stats = SummaryStats()
+        stats.reject_count += 1
         return 0
 
     ################################################################################
@@ -2452,7 +2533,7 @@ distribution."""
         """
         Cnf = Config()
         anyversion = None
-        anysuite = [suite] + Cnf.ValueList("Suite::%s::VersionChecks::Enhances" % (suite))
+        anysuite = [suite] + [ vc.reference.suite_name for vc in get_version_checks(suite, "Enhances") ]
         for (s, v) in sv_list:
             if s in [ x.lower() for x in anysuite ]:
                 if not anyversion or apt_pkg.VersionCompare(anyversion, v) <= 0:
@@ -2482,8 +2563,14 @@ distribution."""
 
         # Check versions for each target suite
         for target_suite in self.pkg.changes["distribution"].keys():
-            must_be_newer_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeNewerThan" % (target_suite)) ]
-            must_be_older_than = [ i.lower() for i in cnf.ValueList("Suite::%s::VersionChecks::MustBeOlderThan" % (target_suite)) ]
+            # Check we can find the target suite
+            ts = get_suite(target_suite)
+            if ts is None:
+                self.rejects.append("Cannot find target suite %s to perform version checks" % target_suite)
+                continue
+
+            must_be_newer_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeNewerThan") ]
+            must_be_older_than = [ vc.reference.suite_name for vc in get_version_checks(target_suite, "MustBeOlderThan") ]
 
             # Enforce "must be newer than target suite" even if conffile omits it
             if target_suite not in must_be_newer_than:
@@ -2684,12 +2771,15 @@ distribution."""
                     orig_files[dsc_name]["path"] = old_file
                     orig_files[dsc_name]["location"] = x.location.location_id
                 else:
-                    # TODO: Record the queues and info in the DB so we don't hardcode all this crap
+                    # TODO: Determine queue list dynamically
                     # Not there? Check the queue directories...
-                    for directory in [ "New", "Byhand", "ProposedUpdates", "OldProposedUpdates", "Embargoed", "Unembargoed" ]:
-                        if not Cnf.has_key("Dir::Queue::%s" % (directory)):
+                    for queue_name in [ "byhand", "new", "proposedupdates", "oldproposedupdates", "embargoed", "unembargoed" ]:
+                        queue = get_policy_queue(queue_name, session)
+                        if not queue:
                             continue
-                        in_otherdir = os.path.join(Cnf["Dir::Queue::%s" % (directory)], dsc_name)
+
+                        in_otherdir = os.path.join(queue.path, dsc_name)
+
                         if os.path.exists(in_otherdir):
                             in_otherdir_fh = utils.open_file(in_otherdir)
                             actual_md5 = apt_pkg.md5sum(in_otherdir_fh)
@@ -2736,10 +2826,10 @@ distribution."""
                     source_epochless_version = re_no_epoch.sub('', source_version)
                     dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
                     found = False
-                    for q in ["Embargoed", "Unembargoed", "Newstage"]:
-                        if cnf.has_key("Dir::Queue::%s" % (q)):
-                            if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
-                                found = True
+                    for queue_name in ["embargoed", "unembargoed", "newstage"]:
+                        queue = get_policy_queue(queue_name, session)
+                        if queue and os.path.exists(os.path.join(queue.path, dsc_filename)):
+                            found = True
                     if not found:
                         self.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f))